Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: enhance extractor generation with previous code context (#3712)
tomsideguidef18126e24e97
diff --git a/apps/api/src/lib/deterministicJson/extract.ts b/apps/api/src/lib/deterministicJson/extract.tsindex 12549c92f..2af8bf578 100644--- a/apps/api/src/lib/deterministicJson/extract.ts+++ b/apps/api/src/lib/deterministicJson/extract.ts@@ -49,7 +49,10 @@ export async function extractDeterministicJson(let anchorHtml: string | undefined;- const generate = async (rejectionFeedback?: string): Promise<string> => {+ const generate = async (+ rejectionFeedback?: string,+ previousCode?: string,+ ): Promise<string> => {anchorHtml ??= await buildAnchorContext({html,@@ -66,6 +69,7 @@ export async function extractDeterministicJson(schemaJson,prompt,rejectionFeedback,+ previousCode,},costTracking,);@@ -94,7 +98,7 @@ export async function extractDeterministicJson(`extractor has ${broken.length} too-strict selector(s); regenerating once`,);try {- const repaired = await generate(tooStrictFeedback(broken));+ const repaired = await generate(tooStrictFeedback(broken), code);if (tooStrictSelectors(repaired, html).length < broken.length) {return await run(repaired);}@@ -122,7 +126,7 @@ export async function extractDeterministicJson(log("extractor run failed, regenerating once:", errorMessage(err));// Let a second failure propagate; the caller warns. An empty shape here// would instead read as "page had no data" and hide the real failure.- return await run(await generate(errorMessage(err)));+ return await run(await generate(errorMessage(err), code));}}diff --git a/apps/api/src/lib/deterministicJson/llm/client.ts b/apps/api/src/lib/deterministicJson/llm/client.tsindex c0d339ab7..48a657201 100644--- a/apps/api/src/lib/deterministicJson/llm/client.ts+++ b/apps/api/src/lib/deterministicJson/llm/client.ts@@ -59,7 +59,7 @@ export async function generateCode(res = await generateText({model: getModel(CODEGEN_MODEL, "vertex"),messages,- temperature: 0,+ temperature: 1.0, // even though it would be nice to have more deterministic output, google recommends keeping this at 1 for complex tasksmaxOutputTokens: CODEGEN_MAX_TOKENS,});} catch (err) {@@ -222,6 +222,14 @@ export function makeAskLlm(continue;}if (!raw) {+ // For the text path an empty reply is the intended "absent" answer+ // (ASK_LLM_SYSTEM tells the model to return an empty string), so honor+ // it instead of retrying to a null. Structured calls still need+ // parseable JSON, so an empty body there remains a retryable failure.+ if (!structured) {+ await cache.setLlm(cacheKey, "");+ return "";+ }lastError = "empty response";continue;}diff --git a/apps/api/src/lib/deterministicJson/llm/prompts.ts b/apps/api/src/lib/deterministicJson/llm/prompts.tsindex 3e3d3b7d6..4414790c6 100644--- a/apps/api/src/lib/deterministicJson/llm/prompts.ts+++ b/apps/api/src/lib/deterministicJson/llm/prompts.ts@@ -9,8 +9,9 @@ export const ASK_LLM_SYSTEM = ["You are a text-processing function in a data pipeline.","Return only the requested value: no preamble, no markdown.","You may summarize, classify, translate, or reformat the supplied text,",- "but stay grounded in it: never invent facts or do outside lookups",- 'If the text lacks what you need, return null (or "" / [] per the schema).',+ "but stay grounded in it: never invent facts or do outside lookups.",+ 'If a JSON schema is provided and the text lacks what you need, return null, "" or [] as allowed by the schema.',+ "If no schema is provided and the text lacks what you need, return an empty string.",].join(" ");// Phase 1: pick short verbatim snippets from the markdown that sit next to the@@ -20,8 +21,12 @@ const ANCHOR_PICKER_SYSTEM = `You are a planning step in a web-extraction pipeliRules:- Each snippet must appear VERBATIM in the markdown.-- Prefer short (5-80 char) distinctive phrases: a label ("Balance:"), a value next to a label, a heading, a unique table cell.+- Prefer short (5-80 char) distinctive phrases.+- Prefer stable labels, headings, column names, semantic section text, or nearby static text over sample-specific values.+- Use values as snippets only when they are the most distinctive way to locate the relevant region in the sample page.- Pick one snippet near each field the schema asks for. For a list/array schema, pick 2-4 snippets that hit DIFFERENT items.+- Prefer coverage of distinct schema fields over multiple snippets for the same field.+- Return at most 12 snippets.- Avoid generic phrases ("Home", "Menu", "Read more") and very long quotes.- Skip fields whose data clearly is not on the page.@@ -40,8 +45,14 @@ const EXTRACTOR_SYSTEM = `You write ONE reusable JavaScript function that extracThis function is generated once from ONE sample page, cached, then re-run on OTHER pages of the same shape. Therefore:- Read every value from the DOM at runtime. NEVER return a literal you saw in the sample page - the values differ on other pages. Literals are only allowed in selectors, attribute names, regexes, and the returned object's keys.-- Always return the schema's shape, even on 404 / login / empty pages: use null or [] for anything absent. NEVER throw. A fabricated value is worse than an empty one.+- The returned value must match the target JSON Schema exactly: include required fields, use the correct types, and do not add extra keys.+- For required schema fields or fields central to the task, throw if the expected anchor/container is missing.+- Return null or [] only when the surrounding container was found and the specific optional value is genuinely absent. Do not treat a missing primary container as optional absence.+- Do NOT hide breakage. Use one primary extraction strategy per value. Do not silently fall back to broader whole-page scraping when it fails. If you handle known explicit variants, branch on a specific container or page-state check, and throw when none match.+- Anchor every value to a specific element. Never fish for a value by matching a pattern (e.g. a price, a date, an email) over doc.body.innerText or the whole page - you will match an unrelated occurrence and return a confident, wrong value.+- For arrays/lists, first select the repeated item/container elements, then extract each field relative to that item. Do not query the whole document for per-item fields inside the loop.- When a value is present, coerce it to the schema's type (e.g. strip non-numerics for numbers). Keep dates as the page displays them unless the task asks for a specific format.+- Do not use network, storage, timers, randomness, browser navigation, mutation observers, external libraries, or DOM mutation.Selectors:- Anchor on stable, meaningful things: labels, semantic attributes (itemprop, role, data-*), heading text. Avoid fragile position (:nth-child, sibling chains).@@ -65,7 +76,21 @@ export function buildAnchorPickerMessages(args: {{ role: "system", content: ANCHOR_PICKER_SYSTEM },{role: "user",- content: `### Page markdown\n\n${args.markdownPreview}\n\n### Target JSON Schema\n\n${args.schemaJson}\n\n### Task\n\n${args.userPrompt}\n\n### Output\n\nReturn {"snippets": [...]} with short verbatim fragments from the markdown above.`,+ content: `### Page markdown++${args.markdownPreview}++### Target JSON Schema++${args.schemaJson}++### Task++${args.userPrompt}++### Output++Return {"snippets": [...]} with short verbatim fragments from the markdown above.`,},];}@@ -76,22 +101,56 @@ export function buildExtractorMessages(args: {markdownPreview: string;anchorHtml: string;rejectionFeedback?: string;+ previousCode?: string;}): ChatMessage[] {// Bulky reference first, the task last - long-context models answer better// when the question follows the documents it refers to.const markdown = args.markdownPreview.trim()- ? `### Page markdown (one sample page - the values shown are NOT your output)\n\n${args.markdownPreview}\n\n`+ ? `### Page markdown (one sample page - the values shown are NOT your output)++${args.markdownPreview}++`: "";+ // When we have the prior extractor, ask for a minimal repair rather than a+ // cold rewrite: a page usually drifts in one selector, so the fix should keep+ // the field mappings and structure identical and touch only what broke.+ const fix = args.previousCode?.trim()+ ? `Here is that function:++\`\`\`js+${args.previousCode.trim()}+\`\`\`++Repair it: change only what is needed to fix the issue above - usually a single selector or parse step. Keep the field mappings, structure, and everything that still works identical; do not rewrite from scratch.`+ : `Write the function again, fixing the issue above.`;+const retry = args.rejectionFeedback?.trim()- ? `\n\n### Your previous attempt was rejected\n\n${args.rejectionFeedback}\n\nRewrite the function to fix this. Every value must still come from a runtime DOM read or askLlm, and the result must match the schema exactly.`+ ? `++### Your previous attempt was rejected++${args.rejectionFeedback}++${fix} Every value must still come from a runtime DOM read or askLlm, and the result must match the schema exactly.`: "";return [{ role: "system", content: EXTRACTOR_SYSTEM },{role: "user",- content: `${markdown}### HTML snippets around the target fields\n\n${args.anchorHtml}\n\n### Target JSON Schema\n\n${args.schemaJson}\n\n### Task\n\n${args.userPrompt}${retry}`,+ content: `${markdown}### HTML snippets around the target fields++${args.anchorHtml}++### Target JSON Schema++${args.schemaJson}++### Task++${args.userPrompt}${retry}`,},];}diff --git a/apps/api/src/lib/deterministicJson/pipeline/generate.ts b/apps/api/src/lib/deterministicJson/pipeline/generate.tsindex b40f9e65d..e4e1be1b3 100644--- a/apps/api/src/lib/deterministicJson/pipeline/generate.ts+++ b/apps/api/src/lib/deterministicJson/pipeline/generate.ts@@ -24,10 +24,12 @@ export async function generateExtractor(schemaJson: string;prompt: string;rejectionFeedback?: string;+ previousCode?: string;},costTracking: CostTracking,): Promise<string> {let feedback = args.rejectionFeedback;+ let previousCode = args.previousCode;let lastIssues: GeneratedCodeIssue[] = [];for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {@@ -38,6 +40,7 @@ export async function generateExtractor(markdownPreview: args.markdownPreview,anchorHtml: args.anchorHtml,rejectionFeedback: feedback,+ previousCode,}),costTracking,);@@ -68,6 +71,8 @@ export async function generateExtractor(lastIssues = issues;feedback = formatGeneratedCodeIssues(issues);+ // Repair our own latest attempt next round, not the stale code passed in.+ if (code) previousCode = code;log(`rejecting extractor attempt ${attempt}: ${issues.length} issue(s)\n${feedback}`,);