Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: backfill metadata title from og:title or twitter:title when missing (#2650)
Himanshu Guptacca56034cd86
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex db6505dd4..d0cd70a57 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -292,6 +292,22 @@ fn _extract_metadata(}}+ // Backfill title from og:title, twitter:title, or meta[name="title"] if primary extraction failed+ if !out.contains_key("title") {+ let fallback_title = out+ .get("ogTitle")+ .or_else(|| out.get("og:title"))+ .or_else(|| out.get("twitter:title"))+ .and_then(|v| match v {+ Value::String(s) if !s.is_empty() => Some(s.clone()),+ _ => None,+ });++ if let Some(title) = fallback_title {+ out.insert("title".to_string(), Value::String(title));+ }+ }+Ok(out)}diff --git a/apps/api/src/lib/__tests__/html-transformer.test.ts b/apps/api/src/lib/__tests__/html-transformer.test.tsindex 8e93e3f82..cee16f015 100644--- a/apps/api/src/lib/__tests__/html-transformer.test.ts+++ b/apps/api/src/lib/__tests__/html-transformer.test.ts@@ -261,6 +261,51 @@ describe("HTML Transformer", () => {const metadata = await extractMetadata(html);expect(metadata).toBeDefined();});++ it("should backfill title from og:title when title tag is missing", async () => {+ const html = `+ <html>+ <head>+ <meta property="og:title" content="OpenGraph Title">+ <meta name="description" content="Page description">+ </head>+ <body></body>+ </html>+ `;+ const metadata = await extractMetadata(html);+ expect(metadata.title).toBe("OpenGraph Title");+ expect(metadata.ogTitle).toBe("OpenGraph Title");+ });++ it("should backfill title from twitter:title when title and og:title are missing", async () => {+ const html = `+ <html>+ <head>+ <meta name="twitter:title" content="Twitter Title">+ <meta name="description" content="Page description">+ </head>+ <body></body>+ </html>+ `;+ const metadata = await extractMetadata(html);+ expect(metadata.title).toBe("Twitter Title");+ });++ it("should not backfill title when title tag exists", async () => {+ const html = `+ <html>+ <head>+ <title>Primary Title</title>+ <meta property="og:title" content="OpenGraph Title">+ <meta name="twitter:title" content="Twitter Title">+ </head>+ <body></body>+ </html>+ `;+ const metadata = await extractMetadata(html);+ expect(metadata.title).toBe("Primary Title");+ expect(metadata.ogTitle).toBe("OpenGraph Title");+ });});describe("transformHtml", () => {