Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(go-html-to-md): enhance markdown conversion with robust PRE and … (#2321)

Rafael Miller8797ee32a467
diff --git a/apps/api/sharedLibs/go-html-to-md/go.mod b/apps/api/sharedLibs/go-html-to-md/go.mod
index 99da57619..5db5ba7b3 100644
--- a/apps/api/sharedLibs/go-html-to-md/go.mod
+++ b/apps/api/sharedLibs/go-html-to-md/go.mod
@@ -4,13 +4,15 @@ go 1.23.0
toolchain go1.24.0
-require github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3
+require (
+ github.com/PuerkitoBio/goquery v1.10.3
+ github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3
+ golang.org/x/net v0.41.0
+)
require (
- github.com/PuerkitoBio/goquery v1.10.3 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/kr/pretty v0.3.0 // indirect
- golang.org/x/net v0.41.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)
diff --git a/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go b/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go
index efc137a8c..88dccfe39 100644
--- a/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go
+++ b/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go
@@ -5,11 +5,17 @@ package main
*/
import "C"
import (
+ "strings"
"unsafe"
+
// "log"
+ "unicode/utf8"
+
+ "github.com/PuerkitoBio/goquery"
md "github.com/firecrawl/html-to-markdown"
"github.com/firecrawl/html-to-markdown/plugin"
+ "golang.org/x/net/html"
)
//export ConvertHTMLToMarkdown
@@ -17,6 +23,8 @@ func ConvertHTMLToMarkdown(html *C.char) *C.char {
converter := md.NewConverter("", true, nil)
converter.Use(plugin.GitHubFlavored())
+ addGenericPreRule(converter)
+
markdown, err := converter.ConvertString(C.GoString(html))
if err != nil {
// log.Fatal(err)
@@ -32,3 +40,115 @@ func FreeCString(s *C.char) {
func main() {
// This function is required for the main package
}
+
+// addGenericPreRule adds a robust PRE handler that extracts nested code text
+// (e.g., tables/rows/gutters) and outputs fenced blocks with detected language.
+func addGenericPreRule(conv *md.Converter) {
+ isGutter := func(class string) bool {
+ c := strings.ToLower(class)
+ return strings.Contains(c, "gutter") || strings.Contains(c, "line-numbers")
+ }
+
+ detectLang := func(sel *goquery.Selection) string {
+ classes := sel.AttrOr("class", "")
+ lower := strings.ToLower(classes)
+ for _, part := range strings.Fields(lower) {
+ if strings.HasPrefix(part, "language-") {
+ return strings.TrimPrefix(part, "language-")
+ }
+ if strings.HasPrefix(part, "lang-") {
+ return strings.TrimPrefix(part, "lang-")
+ }
+ }
+ return ""
+ }
+
+ // Collect text recursively; insert newlines after block elements and br
+ var collect func(n *html.Node, b *strings.Builder)
+ collect = func(n *html.Node, b *strings.Builder) {
+ if n == nil {
+ return
+ }
+ switch n.Type {
+ case html.TextNode:
+ b.WriteString(n.Data)
+ case html.ElementNode:
+ name := strings.ToLower(n.Data)
+ // Skip gutters
+ if name != "" {
+ // check class attr for gutters
+ for _, a := range n.Attr {
+ if a.Key == "class" && isGutter(a.Val) {
+ return
+ }
+ }
+ }
+
+ if name == "br" {
+ b.WriteString("\n")
+ }
+
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ collect(c, b)
+ }
+
+ // Newline after block-ish wrappers to preserve lines
+ switch name {
+ case "p", "div", "li", "tr", "table", "thead", "tbody", "tfoot", "section", "article", "blockquote", "pre", "h1", "h2", "h3", "h4", "h5", "h6":
+ b.WriteString("\n")
+ }
+ }
+ }
+
+ conv.AddRules(md.Rule{
+ Filter: []string{"pre"},
+ Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {
+ // find inner <code> if present for language
+ codeSel := selec.Find("code").First()
+ lang := detectLang(codeSel)
+ if lang == "" {
+ lang = detectLang(selec)
+ }
+
+ var b strings.Builder
+ for _, n := range selec.Nodes {
+ collect(n, &b)
+ }
+ content := strings.TrimRight(b.String(), "\n")
+
+ fenceChar, _ := utf8.DecodeRuneInString(opt.Fence)
+ fence := md.CalculateCodeFence(fenceChar, content)
+ text := "\n\n" + fence + lang + "\n" + content + "\n" + fence + "\n\n"
+ return md.String(text)
+ },
+ })
+
+ // Inline code: robustly extract text and fence with backticks
+ conv.AddRules(md.Rule{
+ Filter: []string{"code"},
+ Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {
+ // If inside pre, let the PRE rule handle it
+ if selec.ParentsFiltered("pre").Length() > 0 {
+ return nil
+ }
+ var b strings.Builder
+ for _, n := range selec.Nodes {
+ collect(n, &b)
+ }
+ code := b.String()
+ // collapse multiple newlines for inline code
+ code = md.TrimTrailingSpaces(strings.ReplaceAll(code, "\r\n", "\n"))
+
+ // Choose fence length safely
+ fence := "`"
+ if strings.Contains(code, "`") {
+ fence = "``"
+ if strings.Contains(code, "``") {
+ fence = "```"
+ }
+ }
+ out := fence + code + fence
+ return md.String(out)
+ },
+ })
+}
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index bc7418c35..32cdd8f5b 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -1582,6 +1582,23 @@ describe("Scrape tests", () => {
scrapeTimeout,
);
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "nested code blocks are converted to markdown correctly",
+ async () => {
+ const response = await scrape(
+ {
+ url: `${base}/code-block`,
+ formats: ["markdown"],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.markdown).toContain("MyCustomClient");
+ },
+ scrapeTimeout,
+ );
+
// TODO: check if these are required
describeIf(ALLOW_TEST_SUITE_WEBSITE)(
"__experimental_omceDomain functionality",
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 784103074..356b0b69d 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -517,7 +517,12 @@ const baseScrapeOptions = z
useMock: z.string().optional(),
blockAds: z.boolean().default(true),
proxy: z.enum(["basic", "stealth", "auto"]).default("basic"),
- maxAge: z.number().int().gte(0).safe().default(1 * 24 * 60 * 60 * 1000),
+ maxAge: z
+ .number()
+ .int()
+ .gte(0)
+ .safe()
+ .default(1 * 24 * 60 * 60 * 1000),
storeInCache: z.boolean().default(true),
// @deprecated
__experimental_cache: z.boolean().default(false).optional(),
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 0f393edbc..6bc0875f2 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -49,14 +49,8 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {
});
}
-export async function recordRobotsBlocked(
- crawlId: string,
- url: string,
-) {
- await redisEvictConnection.sadd(
- "crawl:" + crawlId + ":robots_blocked",
- url,
- );
+export async function recordRobotsBlocked(crawlId: string, url: string) {
+ await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);
await redisEvictConnection.expire(
"crawl:" + crawlId + ":robots_blocked",
24 * 60 * 60,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index c3f6448e6..185e1c64c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -67,8 +67,6 @@ async function scrapePDFWithRunPodMU(
maxPages,
});
-
-
if (
process.env.PDF_MU_V2_EXPERIMENT === "true" &&
process.env.PDF_MU_V2_BASE_URL &&
@@ -115,7 +113,6 @@ async function scrapePDFWithRunPodMU(
})();
}
-
const muV1StartedAt = Date.now();
const podStart = await robustFetch({
url:
diff --git a/apps/test-site/src/pages/code-block.astro b/apps/test-site/src/pages/code-block.astro
new file mode 100644
index 000000000..9c96fdec9
--- /dev/null
+++ b/apps/test-site/src/pages/code-block.astro
@@ -0,0 +1,116 @@
+---
+import BaseHead from '../components/BaseHead.astro';
+import Footer from '../components/Footer.astro';
+import Header from '../components/Header.astro';
+---
+
+<!doctype html>
+<html lang="en">
+ <head>
+ <BaseHead title="Inline Code Backticks Test" description="Test page for inline code with backticks" />
+ </head>
+ <body>
+ <Header />
+ <main>
+ <h1>Inline Code Backticks Test</h1>
+ <p>
+ This page tests the conversion of inline code elements that contain backticks.
+ The HTML <code>&lt;code&gt;</code> tag should be converted to markdown without
+ escaping backticks, relying on longer fences instead.
+ </p>
+ <p>
+ Here are some examples:
+ </p>
+ <ul>
+ <li>Command: <code>ls</code> - should render as <code>ls</code> in markdown</li>
+ <li>Template literal: <code>`template`</code> - should render without escaped backticks</li>
+ <li>Nested backticks: <code>``double``</code> - should use longer fence</li>
+ <li>HTML entity: <code>&lt;code&gt;</code> - should handle entity decoding</li>
+ </ul>
+ <p>
+ The key test case is that <code>&lt;code&gt;ls&lt;/code&gt;</code> should
+ render as markdown without literal backslashes before backticks. Specifically,
+ the HTML entity sequence <code>&lt;code&gt;ls&lt;/code&gt;</code> should render
+ as <code>ls</code> in markdown (using longer fences if needed), not as
+ <code>\`ls\`</code> with escaped backticks.
+ </p>
+ <code class="code-block text-sm">
+ <div class="code-block-inner">
+ <table class="code-block-line-group">
+ <colgroup>
+ <col class="w-fit">
+ <col>
+ </colgroup>
+ <tbody>
+ <tr class="code-block-line">
+ <td class="code-block-line-gutter">
+ <span>1</span>
+ </td>
+ <td class="code-block-line-content">
+ <span class="line">client&lt;llm&gt; MyCustomClient {</span>
+ </td>
+ </tr>
+ <tr class="code-block-line">
+ <td class="code-block-line-gutter">
+ <span>2</span>
+ </td>
+ <td class="code-block-line-content">
+ <span class="line"> provider "openai"</span>
+ </td>
+ </tr>
+ <tr class="code-block-line">
+ <td class="code-block-line-gutter">
+ <span>3</span>
+ </td>
+ <td class="code-block-line-content">
+ <span class="line"> options {</span>
+ </td>
+ </tr>
+ <tr class="code-block-line">
+ <td class="code-block-line-gutter">
+ <span>4</span>
+ </td>
+ <td class="code-block-line-content">
+ <span class="line"> model "gpt-5-mini"</span>
+ </td>
+ </tr>
+ <tr class="code-block-line">
+ <td class="code-block-line-gutter">
+ <span>5</span>
+ </td>
+ <td class="code-block-line-content">
+ <span class="line"> // Set the API key using an environment variable</span>
+ </td>
+ </tr>
+ <tr class="code-block-line">
+ <td class="code-block-line-gutter">
+ <span>6</span>
+ </td>
+ <td class="code-block-line-content">
+ <span class="line"> api_key env.MY_SUPER_SECRET_API_KEY</span>
+ </td>
+ </tr>
+ <tr class="code-block-line">
+ <td class="code-block-line-gutter">
+ <span>7</span>
+ </td>
+ <td class="code-block-line-content">
+ <span class="line"> }</span>
+ </td>
+ </tr>
+ <tr class="code-block-line">
+ <td class="code-block-line-gutter">
+ <span>8</span>
+ </td>
+ <td class="code-block-line-content">
+ <span class="line">}</span>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </code>
+ </main>
+ <Footer />
+ </body>
+</html>