Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(go-html-to-md): enhance markdown conversion with robust PRE and … (#2321)
Rafael Miller8797ee32a467
diff --git a/apps/api/sharedLibs/go-html-to-md/go.mod b/apps/api/sharedLibs/go-html-to-md/go.modindex 99da57619..5db5ba7b3 100644--- a/apps/api/sharedLibs/go-html-to-md/go.mod+++ b/apps/api/sharedLibs/go-html-to-md/go.mod@@ -4,13 +4,15 @@ go 1.23.0toolchain go1.24.0-require github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3+require (+ github.com/PuerkitoBio/goquery v1.10.3+ github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3+ golang.org/x/net v0.41.0+)require (- github.com/PuerkitoBio/goquery v1.10.3 // indirectgithub.com/andybalholm/cascadia v1.3.3 // indirectgithub.com/kr/pretty v0.3.0 // indirect- golang.org/x/net v0.41.0 // indirectgopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirectgopkg.in/yaml.v2 v2.4.0 // indirect)diff --git a/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go b/apps/api/sharedLibs/go-html-to-md/html-to-markdown.goindex efc137a8c..88dccfe39 100644--- a/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go+++ b/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go@@ -5,11 +5,17 @@ package main*/import "C"import (+ "strings""unsafe"+// "log"+ "unicode/utf8"++ "github.com/PuerkitoBio/goquery"md "github.com/firecrawl/html-to-markdown""github.com/firecrawl/html-to-markdown/plugin"+ "golang.org/x/net/html")//export ConvertHTMLToMarkdown@@ -17,6 +23,8 @@ func ConvertHTMLToMarkdown(html *C.char) *C.char {converter := md.NewConverter("", true, nil)converter.Use(plugin.GitHubFlavored())+ addGenericPreRule(converter)+markdown, err := converter.ConvertString(C.GoString(html))if err != nil {// log.Fatal(err)@@ -32,3 +40,115 @@ func FreeCString(s *C.char) {func main() {// This function is required for the main package}++// addGenericPreRule adds a robust PRE handler that extracts nested code text+// (e.g., tables/rows/gutters) and outputs fenced blocks with detected language.+func addGenericPreRule(conv *md.Converter) {+ isGutter := func(class string) bool {+ c := strings.ToLower(class)+ return strings.Contains(c, "gutter") || strings.Contains(c, "line-numbers")+ }++ detectLang := func(sel *goquery.Selection) string {+ classes := sel.AttrOr("class", "")+ lower := strings.ToLower(classes)+ for _, part := range strings.Fields(lower) {+ if strings.HasPrefix(part, "language-") {+ return strings.TrimPrefix(part, "language-")+ }+ if strings.HasPrefix(part, "lang-") {+ return strings.TrimPrefix(part, "lang-")+ }+ }+ return ""+ }++ // Collect text recursively; insert newlines after block elements and br+ var collect func(n *html.Node, b *strings.Builder)+ collect = func(n *html.Node, b *strings.Builder) {+ if n == nil {+ return+ }+ switch n.Type {+ case html.TextNode:+ b.WriteString(n.Data)+ case html.ElementNode:+ name := strings.ToLower(n.Data)+ // Skip gutters+ if name != "" {+ // check class attr for gutters+ for _, a := range n.Attr {+ if a.Key == "class" && isGutter(a.Val) {+ return+ }+ }+ }++ if name == "br" {+ b.WriteString("\n")+ }++ for c := n.FirstChild; c != nil; c = c.NextSibling {+ collect(c, b)+ }++ // Newline after block-ish wrappers to preserve lines+ switch name {+ case "p", "div", "li", "tr", "table", "thead", "tbody", "tfoot", "section", "article", "blockquote", "pre", "h1", "h2", "h3", "h4", "h5", "h6":+ b.WriteString("\n")+ }+ }+ }++ conv.AddRules(md.Rule{+ Filter: []string{"pre"},+ Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {+ // find inner <code> if present for language+ codeSel := selec.Find("code").First()+ lang := detectLang(codeSel)+ if lang == "" {+ lang = detectLang(selec)+ }++ var b strings.Builder+ for _, n := range selec.Nodes {+ collect(n, &b)+ }+ content := strings.TrimRight(b.String(), "\n")++ fenceChar, _ := utf8.DecodeRuneInString(opt.Fence)+ fence := md.CalculateCodeFence(fenceChar, content)+ text := "\n\n" + fence + lang + "\n" + content + "\n" + fence + "\n\n"+ return md.String(text)+ },+ })++ // Inline code: robustly extract text and fence with backticks+ conv.AddRules(md.Rule{+ Filter: []string{"code"},+ Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {+ // If inside pre, let the PRE rule handle it+ if selec.ParentsFiltered("pre").Length() > 0 {+ return nil+ }+ var b strings.Builder+ for _, n := range selec.Nodes {+ collect(n, &b)+ }+ code := b.String()+ // collapse multiple newlines for inline code+ code = md.TrimTrailingSpaces(strings.ReplaceAll(code, "\r\n", "\n"))++ // Choose fence length safely+ fence := "`"+ if strings.Contains(code, "`") {+ fence = "``"+ if strings.Contains(code, "``") {+ fence = "```"+ }+ }+ out := fence + code + fence+ return md.String(out)+ },+ })+}diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex bc7418c35..32cdd8f5b 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -1582,6 +1582,23 @@ describe("Scrape tests", () => {scrapeTimeout,);+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "nested code blocks are converted to markdown correctly",+ async () => {+ const response = await scrape(+ {+ url: `${base}/code-block`,+ formats: ["markdown"],+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.markdown).toContain("MyCustomClient");+ },+ scrapeTimeout,+ );+// TODO: check if these are requireddescribeIf(ALLOW_TEST_SUITE_WEBSITE)("__experimental_omceDomain functionality",diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 784103074..356b0b69d 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -517,7 +517,12 @@ const baseScrapeOptions = zuseMock: z.string().optional(),blockAds: z.boolean().default(true),proxy: z.enum(["basic", "stealth", "auto"]).default("basic"),- maxAge: z.number().int().gte(0).safe().default(1 * 24 * 60 * 60 * 1000),+ maxAge: z+ .number()+ .int()+ .gte(0)+ .safe()+ .default(1 * 24 * 60 * 60 * 1000),storeInCache: z.boolean().default(true),// @deprecated__experimental_cache: z.boolean().default(false).optional(),diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 0f393edbc..6bc0875f2 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -49,14 +49,8 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {});}-export async function recordRobotsBlocked(- crawlId: string,- url: string,-) {- await redisEvictConnection.sadd(- "crawl:" + crawlId + ":robots_blocked",- url,- );+export async function recordRobotsBlocked(crawlId: string, url: string) {+ await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);await redisEvictConnection.expire("crawl:" + crawlId + ":robots_blocked",24 * 60 * 60,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex c3f6448e6..185e1c64c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -67,8 +67,6 @@ async function scrapePDFWithRunPodMU(maxPages,});--if (process.env.PDF_MU_V2_EXPERIMENT === "true" &&process.env.PDF_MU_V2_BASE_URL &&@@ -115,7 +113,6 @@ async function scrapePDFWithRunPodMU(})();}-const muV1StartedAt = Date.now();const podStart = await robustFetch({url:diff --git a/apps/test-site/src/pages/code-block.astro b/apps/test-site/src/pages/code-block.astronew file mode 100644index 000000000..9c96fdec9--- /dev/null+++ b/apps/test-site/src/pages/code-block.astro@@ -0,0 +1,116 @@+---+import BaseHead from '../components/BaseHead.astro';+import Footer from '../components/Footer.astro';+import Header from '../components/Header.astro';+---++<!doctype html>+<html lang="en">+ <head>+ <BaseHead title="Inline Code Backticks Test" description="Test page for inline code with backticks" />+ </head>+ <body>+ <Header />+ <main>+ <h1>Inline Code Backticks Test</h1>+ <p>+ This page tests the conversion of inline code elements that contain backticks.+ The HTML <code><code></code> tag should be converted to markdown without+ escaping backticks, relying on longer fences instead.+ </p>+ <p>+ Here are some examples:+ </p>+ <ul>+ <li>Command: <code>ls</code> - should render as <code>ls</code> in markdown</li>+ <li>Template literal: <code>`template`</code> - should render without escaped backticks</li>+ <li>Nested backticks: <code>``double``</code> - should use longer fence</li>+ <li>HTML entity: <code><code></code> - should handle entity decoding</li>+ </ul>+ <p>+ The key test case is that <code><code>ls</code></code> should+ render as markdown without literal backslashes before backticks. Specifically,+ the HTML entity sequence <code><code>ls</code></code> should render+ as <code>ls</code> in markdown (using longer fences if needed), not as+ <code>\`ls\`</code> with escaped backticks.+ </p>+ <code class="code-block text-sm">+ <div class="code-block-inner">+ <table class="code-block-line-group">+ <colgroup>+ <col class="w-fit">+ <col>+ </colgroup>+ <tbody>+ <tr class="code-block-line">+ <td class="code-block-line-gutter">+ <span>1</span>+ </td>+ <td class="code-block-line-content">+ <span class="line">client<llm> MyCustomClient {</span>+ </td>+ </tr>+ <tr class="code-block-line">+ <td class="code-block-line-gutter">+ <span>2</span>+ </td>+ <td class="code-block-line-content">+ <span class="line"> provider "openai"</span>+ </td>+ </tr>+ <tr class="code-block-line">+ <td class="code-block-line-gutter">+ <span>3</span>+ </td>+ <td class="code-block-line-content">+ <span class="line"> options {</span>+ </td>+ </tr>+ <tr class="code-block-line">+ <td class="code-block-line-gutter">+ <span>4</span>+ </td>+ <td class="code-block-line-content">+ <span class="line"> model "gpt-5-mini"</span>+ </td>+ </tr>+ <tr class="code-block-line">+ <td class="code-block-line-gutter">+ <span>5</span>+ </td>+ <td class="code-block-line-content">+ <span class="line"> // Set the API key using an environment variable</span>+ </td>+ </tr>+ <tr class="code-block-line">+ <td class="code-block-line-gutter">+ <span>6</span>+ </td>+ <td class="code-block-line-content">+ <span class="line"> api_key env.MY_SUPER_SECRET_API_KEY</span>+ </td>+ </tr>+ <tr class="code-block-line">+ <td class="code-block-line-gutter">+ <span>7</span>+ </td>+ <td class="code-block-line-content">+ <span class="line"> }</span>+ </td>+ </tr>+ <tr class="code-block-line">+ <td class="code-block-line-gutter">+ <span>8</span>+ </td>+ <td class="code-block-line-content">+ <span class="line">}</span>+ </td>+ </tr>+ </tbody>+ </table>+ </div>+ </code>+ </main>+ <Footer />+ </body>+</html>