An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
chore(fetch): tidy the crawl4ai backend after #1067 (#1072)
Yoshiki Miurae37daec7b5ab
diff --git a/lib/schema/fetch.tsx b/lib/schema/fetch.tsxindex c249fe6c..a8608407 100644--- a/lib/schema/fetch.tsx+++ b/lib/schema/fetch.tsx@@ -13,4 +13,13 @@ export const fetchSchema = z.object({)})+export const crawl4aiFetchSchema = fetchSchema.extend({+ type: z+ .enum(['regular', 'api'])+ .default('regular')+ .describe(+ 'Fetch method: "regular" (default) and "api" behave the same, and both support PDFs'+ )+})+export type PartialInquiry = DeepPartial<typeof fetchSchema>diff --git a/lib/tools/__tests__/fetch.test.ts b/lib/tools/__tests__/fetch.test.tsindex 73a25a8c..325c88e9 100644--- a/lib/tools/__tests__/fetch.test.ts+++ b/lib/tools/__tests__/fetch.test.ts@@ -1,4 +1,5 @@import { afterEach, describe, expect, it, vi } from 'vitest'+import { z } from 'zod'import { createFetchTool, fetchTool } from '@/lib/tools/fetch'import { createCitationLabelAllocator } from '@/lib/utils/citation'@@ -342,6 +343,32 @@ describe('crawl4ai extraction', () => {expect(fetchMock.mock.calls[0]?.[0]).toBe(`${baseUrl}/crawl`)})+ it('sends no Authorization header without an API key', async () => {+ const fetchMock = fetchCrawl4ai(+ () => new Response(JSON.stringify(pageResponse))+ )++ await runCrawl4ai()++ expect(fetchMock.mock.calls[0]?.[1]?.headers).not.toHaveProperty(+ 'Authorization'+ )+ })++ it('tells the model both types read PDFs only when configured', () => {+ const unsupported = 'does NOT support PDFs'+ const describeTool = () => {+ const tool = createFetchTool()+ return `${tool.description} ${JSON.stringify(z.toJSONSchema(tool.inputSchema as z.ZodType))}`+ }++ expect(describeTool()).toContain(unsupported)++ vi.stubEnv('CRAWL4AI_BASE_URL', baseUrl)++ expect(describeTool()).not.toContain(unsupported)+ })+it('sends the API key when one is configured', async () => {vi.stubEnv('CRAWL4AI_API_KEY', 'test-crawl4ai-key')const fetchMock = fetchCrawl4ai(diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.tsindex 418408a1..a9aca0d0 100644--- a/lib/tools/fetch.ts+++ b/lib/tools/fetch.ts@@ -1,7 +1,7 @@import { tool, UIToolInvocation } from 'ai'import { INVALID_URL_SENTINEL, ToolFailureError } from '@/lib/errors/tool-error'-import { fetchSchema } from '@/lib/schema/fetch'+import { crawl4aiFetchSchema, fetchSchema } from '@/lib/schema/fetch'import { SearchResults as SearchResultsType } from '@/lib/types'import {assignCitationLabels,@@ -187,16 +187,9 @@ async function fetchTavilyExtractData(url: string): Promise<SearchResultsType> {}}-// Everything goes through the server's /crawl pipeline, in two deliberate-// passes. The first pass renders the page in a browser with the same fit-// filter the server's /md endpoint pins (DefaultMarkdownGenerator +-// PruningContentFilterLXML), and its response carries the content type and-// metadata the old /md string collapsed away. Chromium cannot render PDFs, so-// anything the first pass fails to extract gets a second pass with the PDF-// scraping strategy, which makes the server swap in its dedicated PDF crawler.-// Detection therefore happens through the configured backend: no request to-// the origin is made from Morphic itself, and a mis-suffixed URL costs one-// fast failed pass rather than misrouting the fetch.+// crawl4ai's /crawl is called in two passes: a browser render first, then the+// PDF scraping strategy for anything the render could not extract, since+// Chromium cannot render PDFs. Morphic itself never requests the origin.interface Crawl4aiCrawlResult {url?: stringredirected_url?: string | null@@ -210,7 +203,6 @@ interface Crawl4aiCrawlResult {}const CRAWL4AI_PAGE_TIMEOUT_MS = 30000-// PDFs download and parse server-side, which is slower than a page render.const CRAWL4AI_PDF_TIMEOUT_MS = 60000async function postCrawl4ai(@@ -222,16 +214,17 @@ async function postCrawl4ai(success?: booleanresults?: Crawl4aiCrawlResult[]}> {+ const apiKey = process.env.CRAWL4AI_API_KEY+try {const response = await fetch(`${baseUrl}/crawl`, {method: 'POST',headers: {- Authorization: `Bearer ${process.env.CRAWL4AI_API_KEY ?? ''}`,+ ...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),'Content-Type': 'application/json'},body: JSON.stringify({ urls: [url], crawler_config: crawlerConfig }),- // AbortSignal.timeout stays armed through body consumption, so a server- // that sends headers and stalls mid-body still hits the deadline.+ // Unlike a cleared timer, this still fires while the body is being read.signal: AbortSignal.timeout(timeoutMs)})@@ -256,9 +249,7 @@ async function postCrawl4ai(}function crawl4aiMarkdown(result: Crawl4aiCrawlResult): string {- // fit_markdown is the pruned read the fit filter produces; it is empty for- // PDFs (nothing to prune) and for sparse pages the pruner rejects, where the- // raw conversion is still worth reading back.+ // fit_markdown is empty for PDFs and for sparse pages the pruner rejects.return result.markdown?.fit_markdown || result.markdown?.raw_markdown || ''}@@ -269,9 +260,7 @@ function crawl4aiSearchResults(): SearchResultsType {const content = markdown.slice(0, CONTENT_CHARACTER_LIMIT)const resultUrl = result.redirected_url || result.url || url- // The server validated its own hops, but this URL is what comes back into- // Morphic for display, so it gets the same public-space gate every URL in a- // result must pass. It throws on a private or malformed reported target.+ // The crawler reports this URL, so it passes the same gate as any result.assertPublicUrl(resultUrl)const rawTitle = result.metadata?.title?.trim() || new URL(resultUrl).hostnameconst title =@@ -296,8 +285,6 @@ function crawl4aiPayload(json: {success?: booleanresults?: Crawl4aiCrawlResult[]}): { success: boolean; result: Crawl4aiCrawlResult; markdown: string } {- // The envelope carries the request-level success; the per-URL result holds- // the markdown, its own success, and any server-side error message.const result = json.results?.[0] ?? {}return {success: Boolean(json.success && result.success),@@ -307,9 +294,6 @@ function crawl4aiPayload(json: {}async function fetchCrawl4aiData(url: string): Promise<SearchResultsType> {- // Strip trailing slashes so `http://crawler:11235/` never degrades into- // `//crawl`: this backend takes over every fetch, so a malformed route- // would break all of them at once.const baseUrl = process.env.CRAWL4AI_BASE_URL?.replace(/\/+$/, '')if (!baseUrl) {throw new Error('CRAWL4AI_BASE_URL is not configured')@@ -346,8 +330,6 @@ async function fetchCrawl4aiData(url: string): Promise<SearchResultsType> {return crawl4aiSearchResults(url, second.result, second.markdown)}- // The second pass is the deliberate look, so its server-side message is the- // most specific; fall back to the first pass's for context.throw new Error(second.result.error_message ||first.result.error_message ||@@ -361,19 +343,16 @@ export function createFetchTool(options?: {const labelAllocator =options?.labelAllocator ?? createCitationLabelAllocator()- // The PDF guidance must match the configured backend: a self-hosted- // crawl4ai instance extracts PDFs on either type, while the plain HTML- // path (and the Jina/Tavily extraction services behind "api") only serve- // PDFs through "api". Without crawl4ai the description is byte-identical- // to the one on main: deployments that never set the variable must not- // see their prompts move with this change.- const description = process.env.CRAWL4AI_BASE_URL- ? 'Fetch content from any URL and get clean markdown back. Use "regular" (default) for web pages and "api" for pages that need advanced handling. PDFs are extracted and converted to text on either type. The "api" type also gives deeper extraction for complex JavaScript-rendered pages, better markdown formatting, and table extraction. When a crawl4ai instance is configured via CRAWL4AI_BASE_URL it serves both types; otherwise "api" uses Jina Reader if available and falls back to Tavily Extract.'+ // crawl4ai serves both types, so its wording differs. The default wording+ // is what every other deployment's prompt carries and must not change here.+ const usesCrawl4ai = !!process.env.CRAWL4AI_BASE_URL+ const description = usesCrawl4ai+ ? 'Fetch content from any URL as clean markdown through a self-hosted crawl4ai instance. Web pages, JavaScript-rendered pages and PDFs are all supported, and the "regular" and "api" types behave the same.': 'Fetch content from any URL. By default uses "regular" type which performs fast, direct HTML fetching without external APIs - ideal for most websites. IMPORTANT: "regular" type does NOT support PDFs and will fail on PDF URLs. Use "api" type when you need: 1) PDF content extraction (required for .pdf URLs), 2) Complex JavaScript-rendered pages, 3) Better markdown formatting, 4) Table extraction. The "api" type requires Jina or Tavily API keys and uses Jina Reader if available, otherwise falls back to Tavily Extract.'return tool({description,- inputSchema: fetchSchema,+ inputSchema: usesCrawl4ai ? crawl4aiFetchSchema : fetchSchema,async *execute({ url, type = 'regular' }) {assertFetchableUrl(url)// Ahead of the branch below, so an address the server must not reach is@@ -396,14 +375,8 @@ export function createFetchTool(options?: {// reading, and an aborted stream is not a failure of the page.try {if (process.env.CRAWL4AI_BASE_URL) {- // A configured crawl4ai instance is the fetch backend for every- // type: it takes over regular fetching too, not just API extraction.- // Settlement before the handoff stops Morphic from pointing the- // crawler at private space; the crawler's own redirect hops and- // connected peers are guarded server-side (seed validation, the- // pinning egress proxy for browser hops, per-hop validators for the- // PDF crawler), and anything it reports back passes the public gate- // again inside fetchCrawl4aiData before it reaches a result.+ // The name is settled before the handoff; redirect hops are the+ // crawler's to guard.await assertResolvedPublicUrl(url)results = await fetchCrawl4aiData(url)} else if (type === 'regular') {@@ -414,8 +387,7 @@ export function createFetchTool(options?: {// be settled before it is handed over rather than at connect time.await assertResolvedPublicUrl(url)- // Use API-based extraction (Jina or Tavily; crawl4ai, when- // configured, already took the branch above for every type)+ // Use API-based extraction (Jina or Tavily)const useJina = process.env.JINA_API_KEYif (useJina) {results = await fetchJinaReaderData(url)diff --git a/lib/utils/safe-fetch.ts b/lib/utils/safe-fetch.tsindex c812f4e6..d4d10b39 100644--- a/lib/utils/safe-fetch.ts+++ b/lib/utils/safe-fetch.ts@@ -243,7 +243,6 @@ export async function assertResolvedPublicUrl(value: string): Promise<void> {}interface SafeFetchInit {- method?: stringheaders?: Record<string, string>signal?: AbortSignal}