Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: enhance extractor generation with previous code context (#3712)

tomsideguidef18126e24e97
diff --git a/apps/api/src/lib/deterministicJson/extract.ts b/apps/api/src/lib/deterministicJson/extract.ts
index 12549c92f..2af8bf578 100644
--- a/apps/api/src/lib/deterministicJson/extract.ts
+++ b/apps/api/src/lib/deterministicJson/extract.ts
@@ -49,7 +49,10 @@ export async function extractDeterministicJson(
let anchorHtml: string | undefined;
- const generate = async (rejectionFeedback?: string): Promise<string> => {
+ const generate = async (
+ rejectionFeedback?: string,
+ previousCode?: string,
+ ): Promise<string> => {
anchorHtml ??= await buildAnchorContext(
{
html,
@@ -66,6 +69,7 @@ export async function extractDeterministicJson(
schemaJson,
prompt,
rejectionFeedback,
+ previousCode,
},
costTracking,
);
@@ -94,7 +98,7 @@ export async function extractDeterministicJson(
`extractor has ${broken.length} too-strict selector(s); regenerating once`,
);
try {
- const repaired = await generate(tooStrictFeedback(broken));
+ const repaired = await generate(tooStrictFeedback(broken), code);
if (tooStrictSelectors(repaired, html).length < broken.length) {
return await run(repaired);
}
@@ -122,7 +126,7 @@ export async function extractDeterministicJson(
log("extractor run failed, regenerating once:", errorMessage(err));
// Let a second failure propagate; the caller warns. An empty shape here
// would instead read as "page had no data" and hide the real failure.
- return await run(await generate(errorMessage(err)));
+ return await run(await generate(errorMessage(err), code));
}
}
diff --git a/apps/api/src/lib/deterministicJson/llm/client.ts b/apps/api/src/lib/deterministicJson/llm/client.ts
index c0d339ab7..48a657201 100644
--- a/apps/api/src/lib/deterministicJson/llm/client.ts
+++ b/apps/api/src/lib/deterministicJson/llm/client.ts
@@ -59,7 +59,7 @@ export async function generateCode(
res = await generateText({
model: getModel(CODEGEN_MODEL, "vertex"),
messages,
- temperature: 0,
+ temperature: 1.0, // even though it would be nice to have more deterministic output, google recommends keeping this at 1 for complex tasks
maxOutputTokens: CODEGEN_MAX_TOKENS,
});
} catch (err) {
@@ -222,6 +222,14 @@ export function makeAskLlm(
continue;
}
if (!raw) {
+ // For the text path an empty reply is the intended "absent" answer
+ // (ASK_LLM_SYSTEM tells the model to return an empty string), so honor
+ // it instead of retrying to a null. Structured calls still need
+ // parseable JSON, so an empty body there remains a retryable failure.
+ if (!structured) {
+ await cache.setLlm(cacheKey, "");
+ return "";
+ }
lastError = "empty response";
continue;
}
diff --git a/apps/api/src/lib/deterministicJson/llm/prompts.ts b/apps/api/src/lib/deterministicJson/llm/prompts.ts
index 3e3d3b7d6..4414790c6 100644
--- a/apps/api/src/lib/deterministicJson/llm/prompts.ts
+++ b/apps/api/src/lib/deterministicJson/llm/prompts.ts
@@ -9,8 +9,9 @@ export const ASK_LLM_SYSTEM = [
"You are a text-processing function in a data pipeline.",
"Return only the requested value: no preamble, no markdown.",
"You may summarize, classify, translate, or reformat the supplied text,",
- "but stay grounded in it: never invent facts or do outside lookups",
- 'If the text lacks what you need, return null (or "" / [] per the schema).',
+ "but stay grounded in it: never invent facts or do outside lookups.",
+ 'If a JSON schema is provided and the text lacks what you need, return null, "" or [] as allowed by the schema.',
+ "If no schema is provided and the text lacks what you need, return an empty string.",
].join(" ");
// Phase 1: pick short verbatim snippets from the markdown that sit next to the
@@ -20,8 +21,12 @@ const ANCHOR_PICKER_SYSTEM = `You are a planning step in a web-extraction pipeli
Rules:
- Each snippet must appear VERBATIM in the markdown.
-- Prefer short (5-80 char) distinctive phrases: a label ("Balance:"), a value next to a label, a heading, a unique table cell.
+- Prefer short (5-80 char) distinctive phrases.
+- Prefer stable labels, headings, column names, semantic section text, or nearby static text over sample-specific values.
+- Use values as snippets only when they are the most distinctive way to locate the relevant region in the sample page.
- Pick one snippet near each field the schema asks for. For a list/array schema, pick 2-4 snippets that hit DIFFERENT items.
+- Prefer coverage of distinct schema fields over multiple snippets for the same field.
+- Return at most 12 snippets.
- Avoid generic phrases ("Home", "Menu", "Read more") and very long quotes.
- Skip fields whose data clearly is not on the page.
@@ -40,8 +45,14 @@ const EXTRACTOR_SYSTEM = `You write ONE reusable JavaScript function that extrac
This function is generated once from ONE sample page, cached, then re-run on OTHER pages of the same shape. Therefore:
- Read every value from the DOM at runtime. NEVER return a literal you saw in the sample page - the values differ on other pages. Literals are only allowed in selectors, attribute names, regexes, and the returned object's keys.
-- Always return the schema's shape, even on 404 / login / empty pages: use null or [] for anything absent. NEVER throw. A fabricated value is worse than an empty one.
+- The returned value must match the target JSON Schema exactly: include required fields, use the correct types, and do not add extra keys.
+- For required schema fields or fields central to the task, throw if the expected anchor/container is missing.
+- Return null or [] only when the surrounding container was found and the specific optional value is genuinely absent. Do not treat a missing primary container as optional absence.
+- Do NOT hide breakage. Use one primary extraction strategy per value. Do not silently fall back to broader whole-page scraping when it fails. If you handle known explicit variants, branch on a specific container or page-state check, and throw when none match.
+- Anchor every value to a specific element. Never fish for a value by matching a pattern (e.g. a price, a date, an email) over doc.body.innerText or the whole page - you will match an unrelated occurrence and return a confident, wrong value.
+- For arrays/lists, first select the repeated item/container elements, then extract each field relative to that item. Do not query the whole document for per-item fields inside the loop.
- When a value is present, coerce it to the schema's type (e.g. strip non-numerics for numbers). Keep dates as the page displays them unless the task asks for a specific format.
+- Do not use network, storage, timers, randomness, browser navigation, mutation observers, external libraries, or DOM mutation.
Selectors:
- Anchor on stable, meaningful things: labels, semantic attributes (itemprop, role, data-*), heading text. Avoid fragile position (:nth-child, sibling chains).
@@ -65,7 +76,21 @@ export function buildAnchorPickerMessages(args: {
{ role: "system", content: ANCHOR_PICKER_SYSTEM },
{
role: "user",
- content: `### Page markdown\n\n${args.markdownPreview}\n\n### Target JSON Schema\n\n${args.schemaJson}\n\n### Task\n\n${args.userPrompt}\n\n### Output\n\nReturn {"snippets": [...]} with short verbatim fragments from the markdown above.`,
+ content: `### Page markdown
+
+${args.markdownPreview}
+
+### Target JSON Schema
+
+${args.schemaJson}
+
+### Task
+
+${args.userPrompt}
+
+### Output
+
+Return {"snippets": [...]} with short verbatim fragments from the markdown above.`,
},
];
}
@@ -76,22 +101,56 @@ export function buildExtractorMessages(args: {
markdownPreview: string;
anchorHtml: string;
rejectionFeedback?: string;
+ previousCode?: string;
}): ChatMessage[] {
// Bulky reference first, the task last - long-context models answer better
// when the question follows the documents it refers to.
const markdown = args.markdownPreview.trim()
- ? `### Page markdown (one sample page - the values shown are NOT your output)\n\n${args.markdownPreview}\n\n`
+ ? `### Page markdown (one sample page - the values shown are NOT your output)
+
+${args.markdownPreview}
+
+`
: "";
+ // When we have the prior extractor, ask for a minimal repair rather than a
+ // cold rewrite: a page usually drifts in one selector, so the fix should keep
+ // the field mappings and structure identical and touch only what broke.
+ const fix = args.previousCode?.trim()
+ ? `Here is that function:
+
+\`\`\`js
+${args.previousCode.trim()}
+\`\`\`
+
+Repair it: change only what is needed to fix the issue above - usually a single selector or parse step. Keep the field mappings, structure, and everything that still works identical; do not rewrite from scratch.`
+ : `Write the function again, fixing the issue above.`;
+
const retry = args.rejectionFeedback?.trim()
- ? `\n\n### Your previous attempt was rejected\n\n${args.rejectionFeedback}\n\nRewrite the function to fix this. Every value must still come from a runtime DOM read or askLlm, and the result must match the schema exactly.`
+ ? `
+
+### Your previous attempt was rejected
+
+${args.rejectionFeedback}
+
+${fix} Every value must still come from a runtime DOM read or askLlm, and the result must match the schema exactly.`
: "";
return [
{ role: "system", content: EXTRACTOR_SYSTEM },
{
role: "user",
- content: `${markdown}### HTML snippets around the target fields\n\n${args.anchorHtml}\n\n### Target JSON Schema\n\n${args.schemaJson}\n\n### Task\n\n${args.userPrompt}${retry}`,
+ content: `${markdown}### HTML snippets around the target fields
+
+${args.anchorHtml}
+
+### Target JSON Schema
+
+${args.schemaJson}
+
+### Task
+
+${args.userPrompt}${retry}`,
},
];
}
diff --git a/apps/api/src/lib/deterministicJson/pipeline/generate.ts b/apps/api/src/lib/deterministicJson/pipeline/generate.ts
index b40f9e65d..e4e1be1b3 100644
--- a/apps/api/src/lib/deterministicJson/pipeline/generate.ts
+++ b/apps/api/src/lib/deterministicJson/pipeline/generate.ts
@@ -24,10 +24,12 @@ export async function generateExtractor(
schemaJson: string;
prompt: string;
rejectionFeedback?: string;
+ previousCode?: string;
},
costTracking: CostTracking,
): Promise<string> {
let feedback = args.rejectionFeedback;
+ let previousCode = args.previousCode;
let lastIssues: GeneratedCodeIssue[] = [];
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
@@ -38,6 +40,7 @@ export async function generateExtractor(
markdownPreview: args.markdownPreview,
anchorHtml: args.anchorHtml,
rejectionFeedback: feedback,
+ previousCode,
}),
costTracking,
);
@@ -68,6 +71,8 @@ export async function generateExtractor(
lastIssues = issues;
feedback = formatGeneratedCodeIssues(issues);
+ // Repair our own latest attempt next round, not the stale code passed in.
+ if (code) previousCode = code;
log(
`rejecting extractor attempt ${attempt}: ${issues.length} issue(s)\n${feedback}`,
);