Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(html-to-markdown): deduplicate linked images with matching href and src (#2977)
Abimael Martell3dd330f075d2
diff --git a/apps/api/sharedLibs/go-html-to-md/go.mod b/apps/api/sharedLibs/go-html-to-md/go.modindex 86bdc140f..ad11b735c 100644--- a/apps/api/sharedLibs/go-html-to-md/go.mod+++ b/apps/api/sharedLibs/go-html-to-md/go.mod@@ -6,7 +6,7 @@ toolchain go1.24.0require (github.com/PuerkitoBio/goquery v1.10.3- github.com/firecrawl/html-to-markdown v0.0.0-20260203173849-25e9840a0878+ github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9golang.org/x/net v0.41.0)@@ -17,4 +17,4 @@ require (gopkg.in/yaml.v2 v2.4.0 // indirect)-replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20260203173849-25e9840a0878+replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9diff --git a/apps/api/sharedLibs/go-html-to-md/go.sum b/apps/api/sharedLibs/go-html-to-md/go.sumindex 8ee3e2473..7ff59160d 100644--- a/apps/api/sharedLibs/go-html-to-md/go.sum+++ b/apps/api/sharedLibs/go-html-to-md/go.sum@@ -5,6 +5,8 @@ github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmggithub.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=github.com/firecrawl/html-to-markdown v0.0.0-20260203173849-25e9840a0878 h1:PsxFhBLrH0KKkYz5FXzUZefuxUQoodu5ZgDyAaDA6p8=github.com/firecrawl/html-to-markdown v0.0.0-20260203173849-25e9840a0878/go.mod h1:jngam+MdNp7FZkhSTlFsuA5hXY21X0+vuiGlpgo2n5o=+github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9 h1:+riTLXKteNT9WJR9sJKh3EL3BwroJxdVAEEl0JZzUTY=+github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9/go.mod h1:jngam+MdNp7FZkhSTlFsuA5hXY21X0+vuiGlpgo2n5o=github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=diff --git a/apps/go-html-to-md-service/go.mod b/apps/go-html-to-md-service/go.modindex 37790320e..9b36831e9 100644--- a/apps/go-html-to-md-service/go.mod+++ b/apps/go-html-to-md-service/go.mod@@ -4,7 +4,7 @@ go 1.23.0require (github.com/PuerkitoBio/goquery v1.10.3- github.com/firecrawl/html-to-markdown v0.0.0-20260203173849-25e9840a0878+ github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9github.com/gorilla/mux v1.8.1github.com/rs/zerolog v1.33.0golang.org/x/net v0.41.0@@ -20,4 +20,4 @@ require (gopkg.in/yaml.v2 v2.4.0 // indirect)-replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20260203173849-25e9840a0878+replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9diff --git a/apps/go-html-to-md-service/go.sum b/apps/go-html-to-md-service/go.sumindex c10bd74bf..86f5c88d4 100644--- a/apps/go-html-to-md-service/go.sum+++ b/apps/go-html-to-md-service/go.sum@@ -6,6 +6,8 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVgithub.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=github.com/firecrawl/html-to-markdown v0.0.0-20260203173849-25e9840a0878 h1:PsxFhBLrH0KKkYz5FXzUZefuxUQoodu5ZgDyAaDA6p8=github.com/firecrawl/html-to-markdown v0.0.0-20260203173849-25e9840a0878/go.mod h1:jngam+MdNp7FZkhSTlFsuA5hXY21X0+vuiGlpgo2n5o=+github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9 h1:+riTLXKteNT9WJR9sJKh3EL3BwroJxdVAEEl0JZzUTY=+github.com/firecrawl/html-to-markdown v0.0.0-20260305014106-d5b7c1a14bb9/go.mod h1:jngam+MdNp7FZkhSTlFsuA5hXY21X0+vuiGlpgo2n5o=github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=