Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import {
ALLOW_TEST_SUITE_WEBSITE,
HAS_AI,
TEST_PRODUCTION,
TEST_SUITE_WEBSITE,
} from "../lib";
import { describeV1, scrape, scrapeTimeout, idmux, Identity } from "./lib";
let identity: Identity;
beforeAll(async () => {
identity = await idmux({
name: "v1-json-extract-format",
concurrency: 100,
credits: 1000000,
});
}, 10000 + scrapeTimeout);
describeV1(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))(
"V1 JSON/Extract Format Backward Compatibility",
() => {
describe("extract format", () => {
it.concurrent(
"should return extracted data in 'extract' field when using format='extract'",
async () => {
const response = await scrape(
{
url: `${TEST_SUITE_WEBSITE}/example.json`,
formats: ["extract"],
extract: {
schema: {
type: "object",
properties: {
userId: { type: "number" },
id: { type: "number" },
title: { type: "string" },
body: { type: "string" },
},
required: ["userId", "id", "title", "body"],
},
},
timeout: scrapeTimeout,
},
identity,
);
// Data should be in extract field, not json field
expect(response.extract).toBeDefined();
expect(response.json).toBeUndefined();
expect(response.extract.userId).toBe(1);
expect(response.extract.id).toBe(1);
expect(response.extract.title).toBeDefined();
expect(response.extract.body).toBeDefined();
},
scrapeTimeout,
);
it.concurrent(
"should work with extract format and custom prompt",
async () => {
const response = await scrape(
{
url: TEST_SUITE_WEBSITE,
formats: ["extract"],
extract: {
prompt: "Extract the main heading from the page",
schema: {
type: "object",
properties: {
mainHeading: { type: "string" },
},
required: ["mainHeading"],
},
},
timeout: scrapeTimeout,
},
identity,
);
// Data should be in extract field, not json field
expect(response.extract).toBeDefined();
expect(response.json).toBeUndefined();
expect(response.extract.mainHeading).toBeDefined();
},
scrapeTimeout * 2,
);
});
describe("json format", () => {
it.concurrent(
"should return extracted data in 'json' field when using format='json'",
async () => {
const response = await scrape(
{
url: `${TEST_SUITE_WEBSITE}/example.json`,
formats: ["json"],
jsonOptions: {
schema: {
type: "object",
properties: {
userId: { type: "number" },
id: { type: "number" },
title: { type: "string" },
body: { type: "string" },
},
required: ["userId", "id", "title", "body"],
},
},
timeout: scrapeTimeout,
},
identity,
);
// Data should be in json field, not extract field
expect(response.json).toBeDefined();
expect(response.extract).toBeUndefined();
expect(response.json.userId).toBe(1);
expect(response.json.id).toBe(1);
expect(response.json.title).toBeDefined();
expect(response.json.body).toBeDefined();
},
scrapeTimeout,
);
it.concurrent(
"should work with json format and custom prompt",
async () => {
const response = await scrape(
{
url: TEST_SUITE_WEBSITE,
formats: ["json"],
jsonOptions: {
prompt: "Extract the main heading from the page",
schema: {
type: "object",
properties: {
mainHeading: { type: "string" },
},
required: ["mainHeading"],
},
},
timeout: scrapeTimeout,
},
identity,
);
// Data should be in json field, not extract field
expect(response.json).toBeDefined();
expect(response.extract).toBeUndefined();
expect(response.json.mainHeading).toBeDefined();
},
scrapeTimeout * 2,
);
});
describe("multiple formats", () => {
it.concurrent(
"should return markdown and extracted data in correct fields",
async () => {
const response = await scrape(
{
url: TEST_SUITE_WEBSITE,
formats: ["markdown", "extract"],
extract: {
schema: {
type: "object",
properties: {
mainHeading: { type: "string" },
hasLinks: { type: "boolean" },
},
required: ["mainHeading", "hasLinks"],
},
},
timeout: scrapeTimeout,
},
identity,
);
// Both markdown and extract should be present
expect(response.markdown).toBeDefined();
expect(response.extract).toBeDefined();
expect(response.json).toBeUndefined();
expect(response.extract.mainHeading).toBeDefined();
expect(typeof response.extract.hasLinks).toBe("boolean");
},
scrapeTimeout,
);
it.concurrent(
"should return markdown and json data in correct fields",
async () => {
const response = await scrape(
{
url: TEST_SUITE_WEBSITE,
formats: ["markdown", "json"],
jsonOptions: {
schema: {
type: "object",
properties: {
mainHeading: { type: "string" },
hasLinks: { type: "boolean" },
},
required: ["mainHeading", "hasLinks"],
},
},
timeout: scrapeTimeout,
},
identity,
);
// Both markdown and json should be present
expect(response.markdown).toBeDefined();
expect(response.json).toBeDefined();
expect(response.extract).toBeUndefined();
expect(response.json.mainHeading).toBeDefined();
expect(typeof response.json.hasLinks).toBe("boolean");
},
scrapeTimeout,
);
});
},
);