Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

20.7 KB
Raw Blame
import {
  createTestIdUrl,
  describeIf,
  ALLOW_TEST_SUITE_WEBSITE,
  TEST_SELF_HOST,
  TEST_SUITE_WEBSITE,
} from "../lib";
import {
  idmux,
  Identity,
  monitorCheckRaw,
  monitorCreateRaw,
  monitorDeleteRaw,
  monitorEmailConfirmRaw,
  monitorEmailConfirmRawViaQuery,
  monitorEmailUnsubscribeRaw,
  monitorGetRaw,
  monitorListRaw,
  monitorPatchRaw,
  monitorRunRaw,
  scrapeTimeout,
} from "./lib";

describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
  let identity: Identity;

  beforeAll(async () => {
    identity = await idmux({
      name: "monitor",
      concurrency: 20,
      credits: 1000000,
    });
  }, 10000);

  it(
    "reserves the known PDF page cost before a repeat monitor run",
    async () => {
      const create = await monitorCreateRaw(
        {
          name: "PDF reservation monitor",
          schedule: { cron: "0 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [
                `${TEST_SUITE_WEBSITE}/example-long.pdf?testId=${crypto.randomUUID()}`,
              ],
              scrapeOptions: { formats: ["markdown"] },
            },
          ],
          notification: { email: { enabled: false } },
        },
        identity,
      );
      expect(create.statusCode).toBe(200);
      const monitorId = create.body.data.id;
      const runToCompletion = async () => {
        const run = await monitorRunRaw(monitorId, identity);
        expect(run.statusCode).toBe(200);
        let check: any;
        for (let i = 0; i < 90; i++) {
          const raw = await monitorCheckRaw(monitorId, run.body.id, identity);
          expect(raw.statusCode).toBe(200);
          check = raw.body.data;
          if (
            ["completed", "partial", "failed", "skipped_no_credits"].includes(
              check.status,
            )
          )
            break;
          await new Promise(resolve => setTimeout(resolve, 1000));
        }
        expect(check.status).toBe("completed");
        return check;
      };
      try {
        const first = await runToCompletion();
        expect(first.estimatedCredits).toBe(1);
        expect(first.actualCredits).toBeGreaterThan(1);
        const second = await runToCompletion();
        expect(second.estimatedCredits).toBe(first.actualCredits);
        expect(second.reservedCredits).toBe(first.actualCredits);
        expect(second.actualCredits).toBe(first.actualCredits);
      } finally {
        await monitorDeleteRaw(monitorId, identity);
      }
    },
    3 * scrapeTimeout,
  );

  it("creates, lists, gets, pauses, and deletes a monitor", async () => {
    const create = await monitorCreateRaw(
      {
        name: "snips monitor",
        schedule: { cron: "*/30 * * * *", timezone: "UTC" },
        targets: [
          {
            type: "scrape",
            urls: [createTestIdUrl(), createTestIdUrl()],
            scrapeOptions: { formats: ["markdown"] },
          },
        ],
        notification: { email: { enabled: false } },
      },
      identity,
    );

    expect(create.statusCode).toBe(200);
    expect(create.body.success).toBe(true);
    expect(create.body.data.id).toEqual(expect.any(String));
    expect(create.body.data.targets[0].id).toEqual(expect.any(String));

    const id = create.body.data.id;
    const list = await monitorListRaw(identity);
    expect(list.statusCode).toBe(200);
    expect(list.body.data.some((x: any) => x.id === id)).toBe(true);

    const get = await monitorGetRaw(id, identity);
    expect(get.statusCode).toBe(200);
    expect(get.body.data.id).toBe(id);

    const patch = await monitorPatchRaw(id, { status: "paused" }, identity);
    expect(patch.statusCode).toBe(200);
    expect(patch.body.data.status).toBe("paused");

    const del = await monitorDeleteRaw(id, identity);
    expect(del.statusCode).toBe(200);
    expect(del.body.success).toBe(true);
  });

  describe("when the team requires zero data retention for search", () => {
    let searchZdr: Identity;

    beforeAll(async () => {
      searchZdr = await idmux({
        name: "monitor/search-zdr",
        credits: 10000,
        flags: { searchZDR: "forced-zdr" },
      });
    }, 10000);

    it("refuses a search monitor", async () => {
      const create = await monitorCreateRaw(
        {
          name: "search zdr monitor",
          schedule: { cron: "0 * * * *", timezone: "UTC" },
          goal: "New releases of the firecrawl SDK",
          targets: [{ type: "search", queries: ["firecrawl sdk release"] }],
          notification: { email: { enabled: false } },
        },
        searchZdr,
      );

      expect(create.statusCode).toBe(400);
      expect(create.body.success).toBe(false);
      expect(create.body.error).toContain("zero data retention");
    });

    it("keeps a scrape monitor and refuses adding a search target to it", async () => {
      const create = await monitorCreateRaw(
        {
          name: "scrape monitor on a search zdr team",
          schedule: { cron: "0 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [createTestIdUrl()],
              scrapeOptions: { formats: ["markdown"] },
            },
          ],
          notification: { email: { enabled: false } },
        },
        searchZdr,
      );
      expect(create.statusCode).toBe(200);
      const id = create.body.data.id;

      try {
        const addSearch = await monitorPatchRaw(
          id,
          {
            goal: "New releases of the firecrawl SDK",
            targets: [{ type: "search", queries: ["firecrawl sdk release"] }],
          },
          searchZdr,
        );
        expect(addSearch.statusCode).toBe(400);
        expect(addSearch.body.error).toContain("zero data retention");

        const pause = await monitorPatchRaw(
          id,
          { status: "paused" },
          searchZdr,
        );
        expect(pause.statusCode).toBe(200);
        expect(pause.body.data.status).toBe("paused");
      } finally {
        await monitorDeleteRaw(id, searchZdr);
      }
    });
  });

  it("accepts an origin field in the create body", async () => {
    const create = await monitorCreateRaw(
      {
        name: "origin monitor",
        schedule: { cron: "*/30 * * * *", timezone: "UTC" },
        targets: [
          {
            type: "scrape",
            urls: [createTestIdUrl()],
          },
        ],
        origin: "[email protected]",
      },
      identity,
    );

    expect(create.statusCode).toBe(200);
    expect(create.body.success).toBe(true);
    const id = create.body.data.id;
    await monitorDeleteRaw(id, identity);
  });

  it("still rejects unknown keys in the create body", async () => {
    const create = await monitorCreateRaw(
      {
        name: "unknown key monitor",
        schedule: { cron: "*/30 * * * *", timezone: "UTC" },
        targets: [
          {
            type: "scrape",
            urls: [createTestIdUrl()],
          },
        ],
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        nonsenseField: "value",
      } as any,
      identity,
    );

    expect(create.statusCode).toBe(400);
    expect(create.body.success).toBe(false);
  });

  it("rejects cron schedules under 15 minutes", async () => {
    const response = await monitorCreateRaw(
      {
        name: "too frequent",
        schedule: { cron: "*/5 * * * *", timezone: "UTC" },
        targets: [
          {
            type: "scrape",
            urls: [createTestIdUrl()],
          },
        ],
      },
      identity,
    );

    expect(response.statusCode).toBe(400);
    expect(response.body.success).toBe(false);
    expect(response.body.error).toContain("15 minutes");
  });

  it(
    "runs a manual scrape monitor check",
    async () => {
      const create = await monitorCreateRaw(
        {
          name: "manual monitor",
          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [createTestIdUrl(), createTestIdUrl()],
              scrapeOptions: { formats: ["markdown"] },
            },
          ],
        },
        identity,
      );
      expect(create.statusCode).toBe(200);

      const monitorId = create.body.data.id;
      const run = await monitorRunRaw(monitorId, identity);
      expect(run.statusCode).toBe(200);
      const checkId = run.body.id;

      let check: any;
      for (let i = 0; i < 90; i++) {
        const raw = await monitorCheckRaw(monitorId, checkId, identity);
        expect(raw.statusCode).toBe(200);
        check = raw.body.data;
        if (["completed", "partial", "failed"].includes(check.status)) break;
        await new Promise(resolve => setTimeout(resolve, 1000));
      }

      expect(["completed", "partial"]).toContain(check.status);
      expect(check.summary.totalPages).toBeGreaterThanOrEqual(2);
      expect(check.pages.length).toBeGreaterThanOrEqual(1);
      expect(check.next).toBeUndefined();

      const firstPage = await monitorCheckRaw(monitorId, checkId, identity, {
        limit: 1,
      });
      expect(firstPage.statusCode).toBe(200);
      expect(firstPage.body.next).toContain("skip=1");
      expect(firstPage.body.next).toContain("limit=1");
      expect(firstPage.body.data.next).toBe(firstPage.body.next);
      expect(firstPage.body.data.pages).toHaveLength(1);

      await monitorDeleteRaw(monitorId, identity);
    },
    2 * scrapeTimeout,
  );

  it(
    "runs a JSON-mode monitor and surfaces a snapshot",
    async () => {
      const jsonSchema = {
        type: "object",
        additionalProperties: false,
        required: ["title"],
        properties: {
          title: {
            type: "string",
            description: "The page title, verbatim.",
          },
        },
      };

      const create = await monitorCreateRaw(
        {
          name: "json-mode monitor",
          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [createTestIdUrl()],
              scrapeOptions: {
                formats: [
                  "markdown",
                  {
                    type: "json",
                    prompt: "Extract the page title verbatim.",
                    schema: jsonSchema,
                  },
                ],
              },
            },
          ],
        },
        identity,
      );
      expect(create.statusCode).toBe(200);

      const monitorId = create.body.data.id;
      const run = await monitorRunRaw(monitorId, identity);
      expect(run.statusCode).toBe(200);
      const checkId = run.body.id;

      let check: any;
      for (let i = 0; i < 90; i++) {
        const raw = await monitorCheckRaw(monitorId, checkId, identity);
        expect(raw.statusCode).toBe(200);
        check = raw.body.data;
        if (["completed", "partial", "failed"].includes(check.status)) break;
        await new Promise(resolve => setTimeout(resolve, 1000));
      }
      expect(["completed", "partial"]).toContain(check.status);
      // First run: status is "new" (no previous scrape to diff against), no
      // snapshot persisted to GCS. We can't assert snapshot.json here
      // without a mutating fixture; a second run with a changed page would
      // be needed. The contract assertion is: when JSON mode is requested,
      // the monitor doesn't fall through the markdown path and crash.
      expect(check.pages[0].status).toBe("new");

      await monitorDeleteRaw(monitorId, identity);
    },
    2 * scrapeTimeout,
  );

  it(
    "runs a deterministicJson-mode monitor target",
    async () => {
      const jsonSchema = {
        type: "object",
        additionalProperties: false,
        required: ["title"],
        properties: {
          title: {
            type: "string",
            description: "The page title, verbatim.",
          },
        },
      };

      const create = await monitorCreateRaw(
        {
          name: "deterministic-json monitor",
          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [createTestIdUrl()],
              scrapeOptions: {
                formats: [
                  "markdown",
                  {
                    type: "deterministicJson",
                    prompt: "Extract the page title verbatim.",
                    schema: jsonSchema,
                  },
                ],
              },
            },
          ],
        },
        identity,
      );
      expect(create.statusCode).toBe(200);

      const monitorId = create.body.data.id;
      const run = await monitorRunRaw(monitorId, identity);
      expect(run.statusCode).toBe(200);
      const checkId = run.body.id;

      let check: any;
      for (let i = 0; i < 90; i++) {
        const raw = await monitorCheckRaw(monitorId, checkId, identity);
        expect(raw.statusCode).toBe(200);
        check = raw.body.data;
        if (["completed", "partial", "failed"].includes(check.status)) break;
        await new Promise(resolve => setTimeout(resolve, 1000));
      }
      // Contract: deterministicJson is accepted, the reusable-json-mode extractor
      // populates document.json via the code-sandbox, and the check flows through
      // the JSON diff path (status "new" on first run) rather than crashing.
      expect(["completed", "partial"]).toContain(check.status);
      expect(check.pages[0].status).toBe("new");

      await monitorDeleteRaw(monitorId, identity);
    },
    2 * scrapeTimeout,
  );

  it(
    "accepts changeTracking-json format on a monitor target",
    async () => {
      const jsonSchema = {
        type: "object",
        additionalProperties: false,
        required: ["title"],
        properties: {
          title: { type: "string", description: "The page title." },
        },
      };

      const create = await monitorCreateRaw(
        {
          name: "ct-json monitor",
          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [createTestIdUrl()],
              scrapeOptions: {
                formats: [
                  "markdown",
                  {
                    type: "changeTracking",
                    modes: ["json"],
                    prompt: "Extract the title.",
                    schema: jsonSchema,
                  },
                ],
              },
            },
          ],
        },
        identity,
      );
      expect(create.statusCode).toBe(200);

      const monitorId = create.body.data.id;
      const run = await monitorRunRaw(monitorId, identity);
      expect(run.statusCode).toBe(200);
      const checkId = run.body.id;

      let check: any;
      for (let i = 0; i < 90; i++) {
        const raw = await monitorCheckRaw(monitorId, checkId, identity);
        expect(raw.statusCode).toBe(200);
        check = raw.body.data;
        if (["completed", "partial", "failed"].includes(check.status)) break;
        await new Promise(resolve => setTimeout(resolve, 1000));
      }
      expect(["completed", "partial"]).toContain(check.status);
      expect(check.pages[0].status).toBe("new");

      await monitorDeleteRaw(monitorId, identity);
    },
    2 * scrapeTimeout,
  );

  describe("email recipient opt-in", () => {
    function externalRecipient(): string {
      const id = Math.random().toString(36).slice(2, 10);
      return `optin-${id}@external-test.example`;
    }

    it("starts external recipients as pending without sending notifications", async () => {
      const recipient = externalRecipient();
      const create = await monitorCreateRaw(
        {
          name: "opt-in monitor",
          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [createTestIdUrl()],
              scrapeOptions: { formats: ["markdown"] },
            },
          ],
          notification: {
            email: { enabled: true, recipients: [recipient] },
          },
        },
        identity,
      );
      expect(create.statusCode).toBe(200);
      expect(create.body.data.emailRecipientSubscriptions).toEqual(
        expect.arrayContaining([
          expect.objectContaining({
            email: recipient.toLowerCase(),
            status: "pending",
            source: "opt_in",
          }),
        ]),
      );

      const get = await monitorGetRaw(create.body.data.id, identity);
      expect(get.body.data.emailRecipientSubscriptions).toEqual(
        expect.arrayContaining([
          expect.objectContaining({
            email: recipient.toLowerCase(),
            status: "pending",
          }),
        ]),
      );

      await monitorDeleteRaw(create.body.data.id, identity);
    });

    it("rejects malformed confirm/unsubscribe tokens with a 400 JSON error", async () => {
      const badConfirm = await monitorEmailConfirmRaw("x");
      expect(badConfirm.statusCode).toBe(400);
      expect(badConfirm.body).toEqual({
        success: false,
        error: "invalid_token",
      });

      const badUnsub = await monitorEmailUnsubscribeRaw("x");
      expect(badUnsub.statusCode).toBe(400);
      expect(badUnsub.body).toEqual({
        success: false,
        error: "invalid_token",
      });
    });

    it("rejects tokens sent in the query string (body-only)", async () => {
      // Tokens in URLs leak into access/proxy logs and APM URL tags, so
      // the controller deliberately ignores query params.
      const unknownToken = "a".repeat(43);
      const response = await monitorEmailConfirmRawViaQuery(unknownToken);
      expect(response.statusCode).toBe(400);
      expect(response.body).toEqual({
        success: false,
        error: "invalid_token",
      });
    });

    it("returns 404 JSON for unknown but well-formed tokens", async () => {
      const unknownToken = "a".repeat(43);

      const confirm = await monitorEmailConfirmRaw(unknownToken);
      expect(confirm.statusCode).toBe(404);
      expect(confirm.body).toEqual({
        success: false,
        error: "not_found",
      });

      const unsub = await monitorEmailUnsubscribeRaw(unknownToken);
      expect(unsub.statusCode).toBe(404);
      expect(unsub.body).toEqual({
        success: false,
        error: "not_found",
      });
    });

    it("does not require opt-in when recipients are unset (team-default path)", async () => {
      const create = await monitorCreateRaw(
        {
          name: "team default monitor",
          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [createTestIdUrl()],
              scrapeOptions: { formats: ["markdown"] },
            },
          ],
          notification: { email: { enabled: true } },
        },
        identity,
      );
      expect(create.statusCode).toBe(200);
      expect(create.body.data.emailRecipientSubscriptions).toEqual([]);

      await monitorDeleteRaw(create.body.data.id, identity);
    });
  });

  it(
    "accepts mixed json + git-diff changeTracking modes",
    async () => {
      const jsonSchema = {
        type: "object",
        additionalProperties: false,
        required: ["title"],
        properties: {
          title: { type: "string", description: "The page title." },
        },
      };

      const create = await monitorCreateRaw(
        {
          name: "ct-mixed monitor",
          schedule: { cron: "*/30 * * * *", timezone: "UTC" },
          targets: [
            {
              type: "scrape",
              urls: [createTestIdUrl()],
              scrapeOptions: {
                formats: [
                  "markdown",
                  {
                    type: "changeTracking",
                    modes: ["json", "git-diff"],
                    prompt: "Extract the title.",
                    schema: jsonSchema,
                  },
                ],
              },
            },
          ],
        },
        identity,
      );
      expect(create.statusCode).toBe(200);

      const monitorId = create.body.data.id;
      const run = await monitorRunRaw(monitorId, identity);
      expect(run.statusCode).toBe(200);
      const checkId = run.body.id;

      let check: any;
      for (let i = 0; i < 90; i++) {
        const raw = await monitorCheckRaw(monitorId, checkId, identity);
        expect(raw.statusCode).toBe(200);
        check = raw.body.data;
        if (["completed", "partial", "failed"].includes(check.status)) break;
        await new Promise(resolve => setTimeout(resolve, 1000));
      }
      expect(["completed", "partial"]).toContain(check.status);
      // First run is always "new"; the assertion here is that mixed-mode
      // configuration is accepted end-to-end without erroring.
      expect(check.pages[0].status).toBe("new");

      await monitorDeleteRaw(monitorId, identity);
    },
    2 * scrapeTimeout,
  );
});