Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

6.8 KB
Raw Blame
import { vi } from "vitest";

vi.mock("ai", async importOriginal => {
  const actual = await importOriginal<typeof import("ai")>();
  return {
    ...actual,
    generateObject: vi.fn(async () => ({
      object: { ok: true },
      usage: { inputTokens: 1, outputTokens: 1 },
    })),
  };
});

import { generateObject } from "ai";
import { encoding_for_model } from "@dqbd/tiktoken";
import type { Mock } from "vitest";
import {
  generateCompletions,
  generateCrawlerOptionsFromPrompt,
} from "./llmExtract";
import { CostTracking } from "../../../lib/cost-tracking";
import { modelPrices } from "../../../lib/extract/usage/model-prices";

const noopLogger = {
  warn: () => {},
  info: () => {},
  error: () => {},
  debug: () => {},
  child: () => noopLogger,
} as any;

function run(markdown: string, modelId: string, schema?: any) {
  return generateCompletions({
    logger: noopLogger,
    options: { schema },
    markdown,
    model: { modelId } as any,
    retryModel: { modelId } as any,
    costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },
    metadata: { teamId: "test-team" },
  });
}

function lastCall() {
  const calls = (generateObject as Mock).mock.calls;
  expect(calls).toHaveLength(1);
  return calls[0][0];
}

function countTokens(text: string): number {
  const encoder = encoding_for_model("gpt-4o-mini");
  try {
    return encoder.encode(text).length;
  } finally {
    encoder.free();
  }
}

beforeEach(() => {
  (generateObject as Mock).mockClear();
});

describe("generateCompletions content trimming", () => {
  // ~3.3 characters per token like real pages: ~400k tokens, three times
  // gpt-4o-mini's window.
  const hugeMarkdown =
    "The quick brown fox jumps over the lazy dog. [link](https://example.com/a/b?c=1) 12345 ".repeat(
      15_000,
    );

  it("trims content that would overflow the model's context window", async () => {
    const result = await run(hugeMarkdown, "gpt-4o-mini");

    expect(result.warning).toContain(
      "the input has been automatically trimmed",
    );
    // The content is trimmed to exactly 80% of the window; the rest of the
    // prompt is a short instruction.
    const budget = Math.floor(
      modelPrices["gpt-4o-mini"].max_input_tokens * 0.8,
    );
    const promptTokens = countTokens(lastCall().prompt);
    expect(promptTokens).toBeGreaterThan(budget);
    expect(promptTokens).toBeLessThanOrEqual(budget + 100);
  });

  it("leaves content that fits untouched", async () => {
    const markdown = "a small page";
    const result = await run(markdown, "gpt-4o-mini");

    expect(result.warning).toBeUndefined();
    expect(lastCall().prompt).toContain(markdown);
  });

  it("does not trim for models without known limits", async () => {
    const result = await run(hugeMarkdown, "some-unlisted-model");

    expect(result.warning).toBeUndefined();
    expect(lastCall().prompt).toContain(hugeMarkdown);
  });
});

describe("generateCompletions schema normalization", () => {
  it("turns a bare property map into an object schema, skipping $schema", async () => {
    await run("page", "gpt-4o-mini", {
      $schema: "http://json-schema.org/draft-07/schema#",
      judgments: {
        type: "array",
        items: {
          type: "object",
          properties: { citation: { type: "String" } },
        },
      },
    });

    expect(lastCall().schema.jsonSchema).toEqual({
      type: "object",
      properties: {
        judgments: {
          type: "array",
          items: {
            type: "object",
            properties: { citation: { type: "string" } },
            required: ["citation"],
            additionalProperties: false,
          },
        },
      },
      required: ["judgments"],
      additionalProperties: false,
    });
  });

  it("wraps nullable root arrays in an object", async () => {
    await run("page", "gpt-4o-mini", {
      type: ["array", "null"],
      items: { type: "string" },
    });

    expect(lastCall().schema.jsonSchema).toEqual({
      type: "object",
      properties: {
        items: { type: ["array", "null"], items: { type: "string" } },
      },
      required: ["items"],
      additionalProperties: false,
    });
  });

  it("closes nullable objects so strict mode accepts them", async () => {
    await run("page", "gpt-4o-mini", {
      type: "object",
      properties: {
        address: {
          type: ["object", "null"],
          properties: { city: { type: "string" } },
        },
      },
    });

    expect(lastCall().schema.jsonSchema.properties.address).toEqual({
      type: ["object", "null"],
      properties: { city: { type: "string" } },
      required: ["city"],
      additionalProperties: false,
    });
  });
});

describe("generateCompletions telemetry metadata", () => {
  it("tags crawl-prompt spans with the crawl id", async () => {
    await generateCrawlerOptionsFromPrompt(
      "only the blog",
      noopLogger,
      new CostTracking(),
      { teamId: "test-team", crawlId: "test-crawl" },
    );

    const telemetry = lastCall().experimental_telemetry;
    expect(telemetry.functionId).toBe("generateCrawlerOptionsFromPrompt");
    expect(telemetry.metadata).toMatchObject({
      teamId: "test-team",
      crawlId: "test-crawl",
    });
  });

  it("leaves crawlId out when previewing params without a crawl", async () => {
    await generateCrawlerOptionsFromPrompt(
      "only the blog",
      noopLogger,
      new CostTracking(),
      { teamId: "test-team" },
    );

    expect(lastCall().experimental_telemetry.metadata).toEqual({
      teamId: "test-team",
    });
  });

  it("keeps tagging scrape spans with the scrape id", async () => {
    await generateCompletions({
      logger: noopLogger,
      options: {},
      markdown: "page",
      costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },
      metadata: { teamId: "test-team", scrapeId: "test-scrape" },
    });

    expect(lastCall().experimental_telemetry.metadata).toMatchObject({
      teamId: "test-team",
      scrapeId: "test-scrape",
    });
  });

  it("adds the requesting feature's span metadata without overriding the job ids", async () => {
    await generateCompletions({
      logger: noopLogger,
      options: {},
      markdown: "page",
      costTrackingOptions: { costTracking: new CostTracking(), metadata: {} },
      metadata: {
        teamId: "test-team",
        functionId: "monitor/searchJudge",
        scrapeId: "test-scrape",
        telemetry: {
          teamId: "other-team",
          jobId: "test-check",
          jobKind: "monitor",
          feature: "monitor_search_judge",
        },
      },
    });

    const telemetry = lastCall().experimental_telemetry;
    expect(telemetry.functionId).toBe("monitor/searchJudge");
    expect(telemetry.metadata).toMatchObject({
      teamId: "test-team",
      scrapeId: "test-scrape",
      jobId: "test-check",
      jobKind: "monitor",
      feature: "monitor_search_judge",
    });
  });
});