Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(markdown-to-html): Use plugin for markdown block parsing (#2785)
Abimael Martellc36a207e2d65
diff --git a/apps/api/sharedLibs/go-html-to-md/go.mod b/apps/api/sharedLibs/go-html-to-md/go.modindex ad11b735c..0d9967c97 100644--- a/apps/api/sharedLibs/go-html-to-md/go.mod+++ b/apps/api/sharedLibs/go-html-to-md/go.mod@@ -6,7 +6,7 @@ toolchain go1.24.0require (github.com/PuerkitoBio/goquery v1.10.3- github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9+ github.com/firecrawl/html-to-markdown v0.0.0-20260305014655-0ec744e89d3cgolang.org/x/net v0.41.0)@@ -17,4 +17,4 @@ require (gopkg.in/yaml.v2 v2.4.0 // indirect)-replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9+replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20260305014655-0ec744e89d3cdiff --git a/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go b/apps/api/sharedLibs/go-html-to-md/html-to-markdown.goindex 88dccfe39..3b7b2c496 100644--- a/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go+++ b/apps/api/sharedLibs/go-html-to-md/html-to-markdown.go@@ -5,25 +5,17 @@ package main*/import "C"import (- "strings""unsafe"- // "log"-- "unicode/utf8"-- "github.com/PuerkitoBio/goquery"md "github.com/firecrawl/html-to-markdown""github.com/firecrawl/html-to-markdown/plugin"- "golang.org/x/net/html")//export ConvertHTMLToMarkdownfunc ConvertHTMLToMarkdown(html *C.char) *C.char {converter := md.NewConverter("", true, nil)converter.Use(plugin.GitHubFlavored())-- addGenericPreRule(converter)+ converter.Use(plugin.RobustCodeBlock())markdown, err := converter.ConvertString(C.GoString(html))if err != nil {@@ -40,115 +32,3 @@ func FreeCString(s *C.char) {func main() {// This function is required for the main package}--// addGenericPreRule adds a robust PRE handler that extracts nested code text-// (e.g., tables/rows/gutters) and outputs fenced blocks with detected language.-func addGenericPreRule(conv *md.Converter) {- isGutter := func(class string) bool {- c := strings.ToLower(class)- return strings.Contains(c, "gutter") || strings.Contains(c, "line-numbers")- }-- detectLang := func(sel *goquery.Selection) string {- classes := sel.AttrOr("class", "")- lower := strings.ToLower(classes)- for _, part := range strings.Fields(lower) {- if strings.HasPrefix(part, "language-") {- return strings.TrimPrefix(part, "language-")- }- if strings.HasPrefix(part, "lang-") {- return strings.TrimPrefix(part, "lang-")- }- }- return ""- }-- // Collect text recursively; insert newlines after block elements and br- var collect func(n *html.Node, b *strings.Builder)- collect = func(n *html.Node, b *strings.Builder) {- if n == nil {- return- }- switch n.Type {- case html.TextNode:- b.WriteString(n.Data)- case html.ElementNode:- name := strings.ToLower(n.Data)- // Skip gutters- if name != "" {- // check class attr for gutters- for _, a := range n.Attr {- if a.Key == "class" && isGutter(a.Val) {- return- }- }- }-- if name == "br" {- b.WriteString("\n")- }-- for c := n.FirstChild; c != nil; c = c.NextSibling {- collect(c, b)- }-- // Newline after block-ish wrappers to preserve lines- switch name {- case "p", "div", "li", "tr", "table", "thead", "tbody", "tfoot", "section", "article", "blockquote", "pre", "h1", "h2", "h3", "h4", "h5", "h6":- b.WriteString("\n")- }- }- }-- conv.AddRules(md.Rule{- Filter: []string{"pre"},- Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {- // find inner <code> if present for language- codeSel := selec.Find("code").First()- lang := detectLang(codeSel)- if lang == "" {- lang = detectLang(selec)- }-- var b strings.Builder- for _, n := range selec.Nodes {- collect(n, &b)- }- content := strings.TrimRight(b.String(), "\n")-- fenceChar, _ := utf8.DecodeRuneInString(opt.Fence)- fence := md.CalculateCodeFence(fenceChar, content)- text := "\n\n" + fence + lang + "\n" + content + "\n" + fence + "\n\n"- return md.String(text)- },- })-- // Inline code: robustly extract text and fence with backticks- conv.AddRules(md.Rule{- Filter: []string{"code"},- Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {- // If inside pre, let the PRE rule handle it- if selec.ParentsFiltered("pre").Length() > 0 {- return nil- }- var b strings.Builder- for _, n := range selec.Nodes {- collect(n, &b)- }- code := b.String()- // collapse multiple newlines for inline code- code = md.TrimTrailingSpaces(strings.ReplaceAll(code, "\r\n", "\n"))-- // Choose fence length safely- fence := "`"- if strings.Contains(code, "`") {- fence = "``"- if strings.Contains(code, "``") {- fence = "```"- }- }- out := fence + code + fence- return md.String(out)- },- })-}diff --git a/apps/go-html-to-md-service/converter.go b/apps/go-html-to-md-service/converter.goindex 0d09bc072..8abbe118f 100644--- a/apps/go-html-to-md-service/converter.go+++ b/apps/go-html-to-md-service/converter.go@@ -1,13 +1,8 @@package mainimport (- "strings"- "unicode/utf8"-- "github.com/PuerkitoBio/goquery"md "github.com/firecrawl/html-to-markdown""github.com/firecrawl/html-to-markdown/plugin"- "golang.org/x/net/html")// Converter handles HTML to Markdown conversion@@ -19,7 +14,7 @@ type Converter struct {func NewConverter() *Converter {converter := md.NewConverter("", true, nil)converter.Use(plugin.GitHubFlavored())- addGenericPreRule(converter)+ converter.Use(plugin.RobustCodeBlock())return &Converter{converter: converter,@@ -31,115 +26,3 @@ func (c *Converter) ConvertHTMLToMarkdown(html string) (string, error) {return c.converter.ConvertString(html)}-// addGenericPreRule adds a robust PRE handler that extracts nested code text-// (e.g., tables/rows/gutters) and outputs fenced blocks with detected language.-func addGenericPreRule(conv *md.Converter) {- isGutter := func(class string) bool {- c := strings.ToLower(class)- return strings.Contains(c, "gutter") || strings.Contains(c, "line-numbers")- }-- detectLang := func(sel *goquery.Selection) string {- classes := sel.AttrOr("class", "")- lower := strings.ToLower(classes)- for _, part := range strings.Fields(lower) {- if strings.HasPrefix(part, "language-") {- return strings.TrimPrefix(part, "language-")- }- if strings.HasPrefix(part, "lang-") {- return strings.TrimPrefix(part, "lang-")- }- }- return ""- }-- // Collect text recursively; insert newlines after block elements and br- var collect func(n *html.Node, b *strings.Builder)- collect = func(n *html.Node, b *strings.Builder) {- if n == nil {- return- }- switch n.Type {- case html.TextNode:- b.WriteString(n.Data)- case html.ElementNode:- name := strings.ToLower(n.Data)- // Skip gutters- if name != "" {- // check class attr for gutters- for _, a := range n.Attr {- if a.Key == "class" && isGutter(a.Val) {- return- }- }- }-- if name == "br" {- b.WriteString("\n")- }-- for c := n.FirstChild; c != nil; c = c.NextSibling {- collect(c, b)- }-- // Newline after block-ish wrappers to preserve lines- switch name {- case "p", "div", "li", "tr", "table", "thead", "tbody", "tfoot", "section", "article", "blockquote", "pre", "h1", "h2", "h3", "h4", "h5", "h6":- b.WriteString("\n")- }- }- }-- conv.AddRules(md.Rule{- Filter: []string{"pre"},- Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {- // find inner <code> if present for language- codeSel := selec.Find("code").First()- lang := detectLang(codeSel)- if lang == "" {- lang = detectLang(selec)- }-- var b strings.Builder- for _, n := range selec.Nodes {- collect(n, &b)- }- content := strings.TrimRight(b.String(), "\n")-- fenceChar, _ := utf8.DecodeRuneInString(opt.Fence)- fence := md.CalculateCodeFence(fenceChar, content)- text := "\n\n" + fence + lang + "\n" + content + "\n" + fence + "\n\n"- return md.String(text)- },- })-- // Inline code: robustly extract text and fence with backticks- conv.AddRules(md.Rule{- Filter: []string{"code"},- Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {- // If inside pre, let the PRE rule handle it- if selec.ParentsFiltered("pre").Length() > 0 {- return nil- }- var b strings.Builder- for _, n := range selec.Nodes {- collect(n, &b)- }- code := b.String()- // collapse multiple newlines for inline code- code = md.TrimTrailingSpaces(strings.ReplaceAll(code, "\r\n", "\n"))-- // Choose fence length safely- fence := "`"- if strings.Contains(code, "`") {- fence = "``"- if strings.Contains(code, "``") {- fence = "```"- }- }- out := fence + code + fence- return md.String(out)- },- })-}-diff --git a/apps/go-html-to-md-service/go.mod b/apps/go-html-to-md-service/go.modindex 9b36831e9..d724ec0a9 100644--- a/apps/go-html-to-md-service/go.mod+++ b/apps/go-html-to-md-service/go.mod@@ -4,10 +4,9 @@ go 1.23.0require (github.com/PuerkitoBio/goquery v1.10.3- github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9+ github.com/firecrawl/html-to-markdown v0.0.0-20260305014655-0ec744e89d3cgithub.com/gorilla/mux v1.8.1github.com/rs/zerolog v1.33.0- golang.org/x/net v0.41.0)require (@@ -15,9 +14,10 @@ require (github.com/kr/pretty v0.3.0 // indirectgithub.com/mattn/go-colorable v0.1.13 // indirectgithub.com/mattn/go-isatty v0.0.20 // indirect+ golang.org/x/net v0.41.0 // indirectgolang.org/x/sys v0.33.0 // indirectgopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirectgopkg.in/yaml.v2 v2.4.0 // indirect)-replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9+replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20260305014655-0ec744e89d3c