Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: noq scrape system (#2419)

delong38a8d64996356
diff --git a/.github/workflows/deploy-go-service.yaml b/.github/workflows/deploy-go-service.yaml
new file mode 100644
index 000000000..3c2b8c729
--- /dev/null
+++ b/.github/workflows/deploy-go-service.yaml
@@ -0,0 +1,42 @@
+name: Deploy Go Service to GHCR
+
+env:
+ DOTNET_VERSION: "6.0.x"
+
+on:
+ push:
+ branches:
+ - main
+ paths:
+ - apps/go-html-to-md-service/**
+ - .github/workflows/deploy-go-service.yaml
+ workflow_dispatch:
+
+jobs:
+ push-app-image:
+ runs-on: ubuntu-latest
+ defaults:
+ run:
+ working-directory: "./apps/go-html-to-md-service"
+ steps:
+ - name: "Checkout GitHub Action"
+ uses: actions/checkout@main
+
+ - name: "Set up Docker Buildx"
+ uses: docker/setup-buildx-action@v3
+
+ - name: "Login to GitHub Container Registry"
+ uses: docker/login-action@v3
+ with:
+ registry: ghcr.io
+ username: ${{github.actor}}
+ password: ${{secrets.GITHUB_TOKEN}}
+
+ - name: "Build and Push Image"
+ uses: docker/build-push-action@v6
+ with:
+ context: ./apps/go-html-to-md-service
+ push: true
+ tags: ghcr.io/firecrawl/go-html-to-md-service:latest
+ cache-from: type=registry,ref=ghcr.io/firecrawl/go-html-to-md-service:buildcache
+ cache-to: type=registry,ref=ghcr.io/firecrawl/go-html-to-md-service:buildcache,mode=max
diff --git a/.gitignore b/.gitignore
index 9a97f69a3..1c2469ae5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,3 +49,6 @@ CLAUDE.local.md
apps/python-sdk/.venv/
/apps/api/running-docs/
+
+
+/apps/go-html-to-md-service/.gomodcache/
\ No newline at end of file
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
index 37d18f12b..d2e122d7d 100644
--- a/apps/api/native/Cargo.toml
+++ b/apps/api/native/Cargo.toml
@@ -14,7 +14,7 @@ kuchikiki = "0.8.2"
lol_html = "2.6.0"
lopdf = "0.38.0"
maud = "0.27.0"
-napi = { version = "3.0.0", features = ["serde-json"] }
+napi = { version = "3.0.0", features = ["serde-json", "tokio_rt"] }
napi-derive = "3.0.0"
nodesig = { git = "https://github.com/firecrawl/nodesig" }
psl = "2.1.140"
@@ -26,6 +26,7 @@ texting_robots = "0.2.2"
url = "2.5.7"
zip = "5.0.0"
calamine = "0.26"
+tokio = "1.48.0"
[build-dependencies]
napi-build = "2"
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index fa8244cf1..d301481ad 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -7,6 +7,7 @@ use std::{
sync::LazyLock,
};
use texting_robots::Robot;
+use tokio::task;
use url::Url;
static FILE_EXTENSIONS: &[&str] = &[
@@ -173,13 +174,7 @@ fn no_sections(url_str: &str) -> bool {
#[inline]
fn is_non_web_protocol(url_str: &str) -> bool {
const NON_WEB_PROTOCOLS: &[&str] = &[
- "mailto:",
- "tel:",
- "telnet:",
- "ftp:",
- "ftps:",
- "ssh:",
- "file:",
+ "mailto:", "tel:", "telnet:", "ftp:", "ftps:", "ssh:", "file:",
];
NON_WEB_PROTOCOLS
@@ -360,10 +355,12 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
&& is_subdomain(&url, &base_url)
{
// When allowing subdomains, still honor include patterns
- let match_target = if data.regex_on_full_url { url_str } else { path };
- if !includes_regex.is_empty()
- && !includes_regex.iter().any(|r| r.is_match(match_target))
- {
+ let match_target = if data.regex_on_full_url {
+ url_str
+ } else {
+ path
+ };
+ if !includes_regex.is_empty() && !includes_regex.iter().any(|r| r.is_match(match_target)) {
denial_reasons.insert(link, INCLUDE_PATTERN.to_string());
continue;
}
@@ -383,9 +380,17 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
/// Filter links based on crawling rules and constraints.
#[napi]
-pub fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {
- _filter_links(data)
- .map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))
+pub async fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {
+ let res = task::spawn_blocking(move || _filter_links(data))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("filter_links join error: {e}"),
+ )
+ })?;
+
+ res.map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))
}
fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, String> {
@@ -553,9 +558,17 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri
/// Filter a single URL based on crawling rules and constraints.
#[napi]
-pub fn filter_url(data: FilterUrlCall) -> Result<FilterUrlResult> {
- _filter_url(data)
- .map_err(|e| Error::new(Status::GenericFailure, format!("Filter URL error: {e}")))
+pub async fn filter_url(data: FilterUrlCall) -> Result<FilterUrlResult> {
+ let res = task::spawn_blocking(move || _filter_url(data))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("filter_url join error: {e}"),
+ )
+ })?;
+
+ res.map_err(|e| Error::new(Status::GenericFailure, format!("Filter URL error: {e}")))
}
fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, String> {
@@ -616,8 +629,17 @@ fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, S
/// Parse XML sitemap content into structured data.
#[napi]
-pub fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {
- _parse_sitemap_xml(&xml_content).map_err(|e| {
+pub async fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {
+ let res = task::spawn_blocking(move || _parse_sitemap_xml(&xml_content))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("parse_sitemap_xml join error: {e}"),
+ )
+ })?;
+
+ res.map_err(|e| {
Error::new(
Status::GenericFailure,
format!("Parse sitemap XML error: {e}"),
@@ -700,13 +722,17 @@ fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingR
/// Process sitemap XML and extract crawling instructions.
#[napi]
-pub fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResult> {
- _process_sitemap(&xml_content).map_err(|e| {
- Error::new(
- Status::GenericFailure,
- format!("Process sitemap error: {e}"),
- )
- })
+pub async fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResult> {
+ let res = task::spawn_blocking(move || _process_sitemap(&xml_content))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("process_sitemap join error: {e}"),
+ )
+ })?;
+
+ res.map_err(|e| Error::new(Status::GenericFailure, format!("Parse sitemap error: {e}")))
}
#[cfg(test)]
@@ -1006,8 +1032,7 @@ mod tests {
// And should exclude blog due to includePaths
assert!(result
.denial_reasons
- .get("https://sub.example.com/blog")
- .is_some());
+ .contains_key("https://sub.example.com/blog"));
assert_eq!(
result
.denial_reasons
diff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rs
index 6d548c3a0..e1afab37f 100644
--- a/apps/api/native/src/html.rs
+++ b/apps/api/native/src/html.rs
@@ -5,6 +5,7 @@ use napi_derive::napi;
use nodesig::{get_node_signature, SignatureMode};
use serde::{Deserialize, Serialize};
use serde_json::Value;
+use tokio::task;
use url::Url;
use crate::utils::to_napi_err;
@@ -12,7 +13,7 @@ use crate::utils::to_napi_err;
fn _extract_base_href_from_document(
document: &NodeRef,
url: &Url,
-) -> Result<String, Box<dyn std::error::Error>> {
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
if let Some(base) = document
.select("base[href]")
.map_err(|_| "Failed to select base href".to_string())?
@@ -27,7 +28,10 @@ fn _extract_base_href_from_document(
Ok(url.to_string())
}
-fn _extract_base_href(html: &str, url: &str) -> Result<String, Box<dyn std::error::Error>> {
+fn _extract_base_href(
+ html: &str,
+ url: &str,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let document = parse_html().one(html);
let url = Url::parse(url)?;
_extract_base_href_from_document(&document, &url)
@@ -35,43 +39,61 @@ fn _extract_base_href(html: &str, url: &str) -> Result<String, Box<dyn std::erro
/// Extract the base href from HTML document.
#[napi]
-pub fn extract_base_href(html: String, url: String) -> napi::Result<String> {
- _extract_base_href(&html, &url).map_err(to_napi_err)
+pub async fn extract_base_href(html: String, url: String) -> napi::Result<String> {
+ let res = task::spawn_blocking(move || _extract_base_href(&html, &url))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("extract_base_href join error: {e}"),
+ )
+ })?;
+
+ res.map_err(to_napi_err)
}
/// Extract all links from HTML document.
#[napi]
-pub fn extract_links(html: Option<String>) -> napi::Result<Vec<String>> {
- let html = match html {
- Some(h) => h,
- None => return Ok(Vec::new()),
- };
+pub async fn extract_links(html: Option<String>) -> napi::Result<Vec<String>> {
+ task::spawn_blocking(move || {
+ let html = match html {
+ Some(h) => h,
+ None => return Ok(Vec::new()),
+ };
- let document = parse_html().one(html.as_str());
+ let document = parse_html().one(html.as_str());
- let anchors: Vec<_> = document
- .select("a[href]")
- .map_err(|_| to_napi_err("Failed to select links"))?
- .collect();
+ let anchors: Vec<_> = document
+ .select("a[href]")
+ .map_err(|_| to_napi_err("Failed to select links"))?
+ .collect();
- let mut out: Vec<String> = Vec::new();
+ let mut out: Vec<String> = Vec::new();
- for anchor in anchors {
- let mut href = match anchor.attributes.borrow().get("href") {
- Some(x) => x.to_string(),
- None => continue,
- };
+ for anchor in anchors {
+ let mut href = match anchor.attributes.borrow().get("href") {
+ Some(x) => x.to_string(),
+ None => continue,
+ };
- if href.starts_with("http:/") && !href.starts_with("http://") {
- href = format!("http://{}", &href[6..]);
- } else if href.starts_with("https:/") && !href.starts_with("https://") {
- href = format!("https://{}", &href[7..]);
- }
+ if href.starts_with("http:/") && !href.starts_with("http://") {
+ href = format!("http://{}", &href[6..]);
+ } else if href.starts_with("https:/") && !href.starts_with("https://") {
+ href = format!("https://{}", &href[7..]);
+ }
- out.push(href);
- }
+ out.push(href);
+ }
- Ok(out)
+ Ok(out)
+ })
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("extract_links join error: {e}"),
+ )
+ })?
}
macro_rules! insert_meta_name {
@@ -112,7 +134,9 @@ macro_rules! insert_meta_property {
};
}
-fn _extract_metadata(html: &str) -> Result<HashMap<String, Value>, Box<dyn std::error::Error>> {
+fn _extract_metadata(
+ html: &str,
+) -> Result<HashMap<String, Value>, Box<dyn std::error::Error + Send + Sync>> {
let document = parse_html().one(html);
let mut out = HashMap::<String, Value>::new();
@@ -267,13 +291,22 @@ fn _extract_metadata(html: &str) -> Result<HashMap<String, Value>, Box<dyn std::
/// Extract metadata from HTML document.
#[napi]
-pub fn extract_metadata(html: Option<String>) -> napi::Result<HashMap<String, Value>> {
- let html = match html {
- Some(h) => h,
- None => return Ok(HashMap::new()),
- };
+pub async fn extract_metadata(html: Option<String>) -> napi::Result<HashMap<String, Value>> {
+ task::spawn_blocking(move || {
+ let html = match html {
+ Some(h) => h,
+ None => return Ok(HashMap::new()),
+ };
- _extract_metadata(&html).map_err(to_napi_err)
+ _extract_metadata(&html).map_err(to_napi_err)
+ })
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("extract_metadata join error: {e}"),
+ )
+ })?
}
const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [
@@ -356,7 +389,9 @@ struct ImageSource {
is_x: bool,
}
-fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn std::error::Error>> {
+fn _transform_html_inner(
+ opts: TransformHtmlOptions,
+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
let mut document = parse_html().one(opts.html.as_ref());
let url = Url::parse(&_extract_base_href_from_document(
&document,
@@ -574,8 +609,17 @@ fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn s
/// Transform and clean HTML content based on provided options.
#[napi]
-pub fn transform_html(opts: TransformHtmlOptions) -> napi::Result<String> {
- _transform_html_inner(opts).map_err(to_napi_err)
+pub async fn transform_html(opts: TransformHtmlOptions) -> napi::Result<String> {
+ let res = task::spawn_blocking(move || _transform_html_inner(opts))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("transform_html join error: {e}"),
+ )
+ })?;
+
+ res.map_err(to_napi_err)
}
fn _get_inner_json(html: &str) -> Result<String, ()> {
@@ -584,8 +628,17 @@ fn _get_inner_json(html: &str) -> Result<String, ()> {
/// Extract inner text content from HTML body.
#[napi]
-pub fn get_inner_json(html: String) -> napi::Result<String> {
- _get_inner_json(&html).map_err(|_| to_napi_err("Failed to get inner JSON"))
+pub async fn get_inner_json(html: String) -> napi::Result<String> {
+ let res = task::spawn_blocking(move || _get_inner_json(&html))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("get_inner_json join error: {e}"),
+ )
+ })?;
+
+ res.map_err(|_| to_napi_err("Failed to get inner JSON"))
}
#[derive(Deserialize, Serialize)]
@@ -612,7 +665,7 @@ pub struct ExtractedAttributeResult {
fn _extract_attributes(
html: &str,
options: &ExtractAttributesOptions,
-) -> Result<Vec<ExtractedAttributeResult>, Box<dyn std::error::Error>> {
+) -> Result<Vec<ExtractedAttributeResult>, Box<dyn std::error::Error + Send + Sync>> {
let document = parse_html().one(html);
let mut results = Vec::new();
@@ -659,21 +712,33 @@ fn _extract_attributes(
/// Extract specified attributes from HTML elements matching selectors.
#[napi]
-pub fn extract_attributes(
+pub async fn extract_attributes(
html: String,
options: ExtractAttributesOptions,
) -> napi::Result<Vec<ExtractedAttributeResult>> {
- _extract_attributes(&html, &options).map_err(to_napi_err)
+ let res = task::spawn_blocking(move || _extract_attributes(&html, &options))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("extract_attributes join error: {e}"),
+ )
+ })?;
+
+ res.map_err(to_napi_err)
}
-fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn std::error::Error>> {
+fn _extract_images(
+ html: &str,
+ base_url: &str,
+) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {
let document = parse_html().one(html);
let base_url = Url::parse(base_url)?;
let base_href = _extract_base_href_from_document(&document, &base_url)?;
let base_href_url = Url::parse(&base_href)?;
let mut images = HashSet::<String>::new();
- let resolve_image_url = |src: &str| -> Result<String, Box<dyn std::error::Error>> {
+ let resolve_image_url = |src: &str| -> Result<String, Box<dyn std::error::Error + Send + Sync>> {
if src.starts_with("data:") || src.starts_with("blob:") {
return Ok(src.to_string());
}
@@ -812,6 +877,98 @@ fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn st
/// Extract all image URLs from HTML document.
#[napi]
-pub fn extract_images(html: String, base_url: String) -> napi::Result<Vec<String>> {
- _extract_images(&html, &base_url).map_err(to_napi_err)
+pub async fn extract_images(html: String, base_url: String) -> napi::Result<Vec<String>> {
+ let res = task::spawn_blocking(move || _extract_images(&html, &base_url))
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("extract_images join error: {e}"),
+ )
+ })?;
+
+ res.map_err(to_napi_err)
+}
+
+/// Process multi-line links in markdown.
+#[napi]
+pub async fn post_process_markdown(markdown: String) -> napi::Result<String> {
+ let res = task::spawn_blocking(move || {
+ let mut link_open_count = 0usize;
+ let mut out = String::with_capacity(markdown.len());
+
+ for ch in markdown.chars() {
+ match ch {
+ '[' => {
+ link_open_count += 1;
+ }
+ ']' => {
+ link_open_count = link_open_count.saturating_sub(1);
+ }
+ _ => {}
+ }
+
+ let inside_link_content = link_open_count > 0;
+ if inside_link_content && ch == '\n' {
+ out.push('\\');
+ out.push('\n');
+ } else {
+ out.push(ch);
+ }
+ }
+
+ remove_skip_to_content_links(&out)
+ })
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("post_process_markdown join error: {e}"),
+ )
+ })?;
+
+ Ok(res)
+}
+
+fn remove_skip_to_content_links(input: &str) -> String {
+ const LABEL: &str = "Skip to Content";
+ let bytes = input.as_bytes();
+ let len = bytes.len();
+ let mut out = String::with_capacity(len);
+ let mut i = 0;
+
+ 'outer: while i < len {
+ if bytes[i] == b'[' {
+ let label_start = i + 1;
+ let label_end = label_start + LABEL.len();
+
+ if label_end <= len && bytes[label_start..label_end].iter().all(|b| b.is_ascii()) {
+ let label_slice = &input[label_start..label_end];
+
+ if label_slice.eq_ignore_ascii_case(LABEL)
+ && label_end + 3 <= len
+ && bytes[label_end] == b']'
+ && bytes[label_end + 1] == b'('
+ && bytes[label_end + 2] == b'#'
+ {
+ let mut j = label_end + 3;
+
+ while j < len {
+ let ch = input[j..].chars().next().unwrap();
+ if ch == ')' {
+ i = j + ch.len_utf8();
+ continue 'outer;
+ }
+ j += ch.len_utf8();
+ }
+ }
+ }
+ }
+
+ let ch = input[i..].chars().next().unwrap();
+ out.push(ch);
+ i += ch.len_utf8();
+ }
+
+ out
}
diff --git a/apps/api/package.json b/apps/api/package.json
index fd68c0474..862388993 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -126,6 +126,7 @@
"pg": "^8.16.3",
"posthog-node": "^4.0.1",
"prettier": "^3.6.2",
+ "prom-client": "^15.1.3",
"psl": "^1.15.0",
"rate-limiter-flexible": "2.4.2",
"redlock": "5.0.0-beta.2",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index c714383ce..abd8388da 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -191,6 +191,9 @@ importers:
prettier:
specifier: ^3.6.2
version: 3.6.2
+ prom-client:
+ specifier: ^15.1.3
+ version: 15.1.3
psl:
specifier: ^1.15.0
version: 1.15.0
@@ -3731,6 +3734,9 @@ packages:
resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}
+ resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
+
resolution: {integrity: sha512-v2YAxEmKaBLahNwE1mjp4WON6huMNeuDvagFZW+ASCuA/ku0bXR9hSMw0XpiqMoA3+rmnyck/tPRSFQkoC9Cuw==}
@@ -5922,6 +5928,10 @@ packages:
resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}
engines: {node: '>=0.4.0'}
+ resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}
+ engines: {node: ^16 || ^18 || >=20}
+
resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}
engines: {node: '>= 6'}
@@ -6465,6 +6475,9 @@ packages:
os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android]
hasBin: true
+ resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
+
resolution: {integrity: sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==}
engines: {node: '>=18'}
@@ -11919,6 +11932,8 @@ snapshots:
dependencies:
file-uri-to-path: 1.0.0
+
@@ -14480,6 +14495,11 @@ snapshots:
+ dependencies:
+ '@opentelemetry/api': 1.9.0
+ tdigest: 0.1.2
+
dependencies:
kleur: 3.0.3
@@ -15067,6 +15087,10 @@ snapshots:
+ dependencies:
+ bintrees: 1.0.2
+
dependencies:
http-proxy-agent: 5.0.0
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index fdbfd086c..23137225b 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -1,7 +1,11 @@
import { parseApi } from "../lib/parseApi";
import { getRateLimiter } from "../services/rate-limiter";
import { AuthResponse, NotificationType, RateLimiterMode } from "../types";
-import { supabase_rr_service, supabase_service } from "../services/supabase";
+import {
+ supabase_acuc_only_service,
+ supabase_rr_service,
+ supabase_service,
+} from "../services/supabase";
import { withAuth } from "../lib/withAuth";
import { RateLimiterRedis } from "rate-limiter-flexible";
import { sendNotification } from "../services/notification/email_notification";
@@ -12,20 +16,7 @@ import { setValue } from "../services/redis";
import { validate } from "uuid";
import * as Sentry from "@sentry/node";
import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types";
-// const { data, error } = await supabase_service
-// .from('api_keys')
-// .select(`
-// key,
-// team_id,
-// teams (
-// subscriptions (
-// price_id
-// )
-// )
-// `)
-// .eq('key', normalizedApi)
-// .limit(1)
-// .single();
+
function normalizedApiIsUuid(potentialUuid: string): boolean {
// Check if the string is a valid UUID
return validate(potentialUuid);
@@ -166,7 +157,10 @@ export async function getACUC(
return acuc;
}
- if (process.env.USE_DB_AUTHENTICATION !== "true") {
+ if (
+ process.env.USE_DB_AUTHENTICATION !== "true" &&
+ !process.env.SUPABASE_ACUC_URL
+ ) {
const acuc = mockACUC();
acuc.is_extract = isExtract;
return acuc;
@@ -187,8 +181,11 @@ export async function getACUC(
let retries = 0;
const maxRetries = 5;
while (retries < maxRetries) {
- const client =
- Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;
+ const client = !!process.env.SUPABASE_ACUC_URL
+ ? supabase_acuc_only_service
+ : Math.random() > 2 / 3
+ ? supabase_rr_service
+ : supabase_service;
({ data, error } = await client.rpc(
"auth_credit_usage_chunk_36",
{
@@ -292,7 +289,10 @@ export async function getACUCTeam(
return acuc;
}
- if (process.env.USE_DB_AUTHENTICATION !== "true") {
+ if (
+ process.env.USE_DB_AUTHENTICATION !== "true" &&
+ !process.env.SUPABASE_ACUC_URL
+ ) {
const acuc = mockACUC();
acuc.is_extract = isExtract;
return acuc;
@@ -314,8 +314,11 @@ export async function getACUCTeam(
const maxRetries = 5;
while (retries < maxRetries) {
- const client =
- Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;
+ const client = !!process.env.SUPABASE_ACUC_URL
+ ? supabase_acuc_only_service
+ : Math.random() > 2 / 3
+ ? supabase_rr_service
+ : supabase_service;
({ data, error } = await client.rpc(
"auth_credit_usage_chunk_36_from_team",
{
@@ -393,6 +396,10 @@ export async function authenticateUser(
res,
mode?: RateLimiterMode,
): Promise<AuthResponse> {
+ if (!!process.env.SUPABASE_ACUC_URL) {
+ return supaAuthenticateUser(req, res, mode);
+ }
+
return withAuth(supaAuthenticateUser, {
success: true,
chunk: null,
diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.ts
index 4174d2372..fbaeb6ae3 100644
--- a/apps/api/src/controllers/v0/admin/metrics.ts
+++ b/apps/api/src/controllers/v0/admin/metrics.ts
@@ -1,6 +1,7 @@
import type { Request, Response } from "express";
import { getRedisConnection } from "../../../services/queue-service";
import { nuqGetLocalMetrics, scrapeQueue } from "../../../services/worker/nuq";
+import { teamConcurrencySemaphore } from "../../../services/worker/team-semaphore";
export async function metricsController(_: Request, res: Response) {
let cursor: string = "0";
@@ -26,6 +27,8 @@ export async function metricsController(_: Request, res: Response) {
}
} while (cursor !== "0");
+ const semaphoreMetrics = await teamConcurrencySemaphore.getMetrics();
+
res.contentType("text/plain").send(`\
# HELP concurrency_limit_queue_job_count The number of jobs in the concurrency limit queue per team
# TYPE concurrency_limit_queue_job_count gauge
@@ -35,9 +38,8 @@ ${Object.entries(metrics)
`concurrency_limit_queue_job_count{team_id="${key}"} ${value}`,
)
.join("\n")}
-
${nuqGetLocalMetrics()}
-`);
+${semaphoreMetrics}`);
}
export async function nuqMetricsController(_: Request, res: Response) {
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index ea3ba4ccc..3d3908884 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -383,6 +383,8 @@ export async function mapController(
});
let result: Awaited<ReturnType<typeof getMapResults>>;
+ let timeoutHandle: NodeJS.Timeout | null = null;
+
const abort = new AbortController();
try {
result = (await Promise.race([
@@ -405,11 +407,12 @@ export async function mapController(
}),
...(req.body.timeout !== undefined
? [
- new Promise((resolve, reject) =>
- setTimeout(() => {
- abort.abort(new MapTimeoutError());
- reject(new MapTimeoutError());
- }, req.body.timeout),
+ new Promise(
+ (_resolve, reject) =>
+ (timeoutHandle = setTimeout(() => {
+ abort.abort(new MapTimeoutError());
+ reject(new MapTimeoutError());
+ }, req.body.timeout)),
),
]
: []),
@@ -424,6 +427,10 @@ export async function mapController(
} else {
throw error;
}
+ } finally {
+ if (timeoutHandle) {
+ clearTimeout(timeoutHandle);
+ }
}
// Bill the team
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 84523d2da..aca00a019 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -8,12 +8,15 @@ import {
ScrapeResponse,
} from "./types";
import { v4 as uuidv4 } from "uuid";
-import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
import { fromV1ScrapeOptions } from "../v2/types";
import { TransportableError } from "../../lib/error";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { NuQJob } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
+import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";
+import { processJobInternal } from "../../services/worker/scrape-worker";
+import { ScrapeJobData } from "../../types";
+import { AbortManagerThrownError } from "../../scraper/scrapeURL/lib/abortManager";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -42,6 +45,7 @@ export async function scrapeController(
const logger = _logger.child({
method: "scrapeController",
jobId,
+ noq: true,
scrapeId: jobId,
teamId: req.auth.team_id,
team_id: req.auth.team_id,
@@ -61,7 +65,7 @@ export async function scrapeController(
const origin = req.body.origin;
const timeout = req.body.timeout;
- const startTime = new Date().getTime();
+ // const startTime = new Date().getTime();
const isDirectToBullMQ =
process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
@@ -73,43 +77,6 @@ export async function scrapeController(
req.auth.team_id,
);
- const jobPriority = await getJobPriority({
- team_id: req.auth.team_id,
- basePriority: 10,
- });
-
- const bullJob = await addScrapeJob(
- {
- url: req.body.url,
- mode: "single_urls",
- team_id: req.auth.team_id,
- scrapeOptions,
- internalOptions: {
- ...internalOptions,
- teamId: req.auth.team_id,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
- ? true
- : false,
- unnormalizedSourceURL: preNormalizedBody.url,
- bypassBilling: isDirectToBullMQ,
- zeroDataRetention,
- teamFlags: req.acuc?.flags ?? null,
- },
- origin,
- integration: req.body.integration,
- startTime: controllerStartTime,
- zeroDataRetention: zeroDataRetention ?? false,
- apiKeyId: req.acuc?.api_key_id ?? null,
- },
- jobId,
- jobPriority,
- isDirectToBullMQ,
- true,
- );
- logger.info(
- "Added scrape job now" + (bullJob ? "" : " (to concurrency queue)"),
- );
-
const totalWait =
(req.body.waitFor ?? 0) +
(req.body.actions ?? []).reduce(
@@ -117,22 +84,80 @@ export async function scrapeController(
0,
);
- let doc: Document;
+ let timeoutHandle: NodeJS.Timeout | null = null;
+ let doc: Document | null = null;
try {
- doc = await waitForJob(
- bullJob ? bullJob : jobId,
- timeout + totalWait,
- zeroDataRetention ?? false,
- logger,
+ const lockStart = Date.now();
+ const aborter = new AbortController();
+ if (timeout) {
+ timeoutHandle = setTimeout(() => {
+ aborter.abort();
+ }, timeout * 0.667);
+ }
+ req.on("close", () => aborter.abort());
+
+ doc = await teamConcurrencySemaphore.withSemaphore(
+ req.auth.team_id,
+ jobId,
+ req.acuc?.concurrency || 1,
+ aborter.signal,
+ timeout ?? 60_000,
+ async () => {
+ const jobPriority = await getJobPriority({
+ team_id: req.auth.team_id,
+ basePriority: 10,
+ });
+
+ const lockTime = Date.now() - lockStart;
+
+ logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {
+ teamId: req.auth.team_id,
+ lockTime,
+ });
+
+ const job: NuQJob<ScrapeJobData> = {
+ id: jobId,
+ status: "active",
+ createdAt: new Date(),
+ priority: jobPriority,
+ data: {
+ url: req.body.url,
+ mode: "single_urls",
+ team_id: req.auth.team_id,
+ scrapeOptions,
+ internalOptions: {
+ ...internalOptions,
+ teamId: req.auth.team_id,
+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
+ ? true
+ : false,
+ unnormalizedSourceURL: preNormalizedBody.url,
+ bypassBilling: isDirectToBullMQ,
+ zeroDataRetention,
+ teamFlags: req.acuc?.flags ?? null,
+ },
+ skipNuq: true,
+ origin,
+ integration: req.body.integration,
+ startTime: controllerStartTime,
+ zeroDataRetention: zeroDataRetention ?? false,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ },
+ };
+
+ const doc = await processJobInternal(job);
+ return doc ?? null;
+ },
);
} catch (e) {
- logger.error(`Error in scrapeController`, {
- version: "v1",
- error: e,
- });
+ const timeoutErr =
+ e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";
- if (zeroDataRetention) {
- await scrapeQueue.removeJob(jobId, logger);
+ if (!timeoutErr) {
+ logger.error(`Error in scrapeController`, {
+ version: "v1",
+ error: e,
+ });
}
if (e instanceof TransportableError) {
@@ -157,12 +182,14 @@ export async function scrapeController(
error: `(Internal server error) - ${e && e.message ? e.message : e}`,
});
}
+ } finally {
+ if (timeoutHandle) {
+ clearTimeout(timeoutHandle);
+ }
}
logger.info("Done with waitForJob");
- await scrapeQueue.removeJob(jobId, logger);
-
logger.info("Removed job from queue");
if (!req.body.formats.includes("rawHtml")) {
@@ -186,7 +213,7 @@ export async function scrapeController(
return res.status(200).json({
success: true,
- data: doc,
+ data: doc!,
scrape_id: origin?.includes("website") ? jobId : undefined,
});
}
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index c3b5e5505..ee7d4692a 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -174,7 +174,15 @@ export async function crawlStatusController(
res: Response<CrawlStatusResponse>,
isBatch = false,
) {
- const isPreviewTeam = req.auth.team_id?.startsWith("preview");
+ const uuidReg =
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+ if (!req.params.jobId || !uuidReg.test(req.params.jobId)) {
+ return res.status(400).json({
+ success: false,
+ error: "Invalid job ID",
+ });
+ }
+
const start =
typeof req.query.skip === "string" ? parseInt(req.query.skip, 10) : 0;
const end =
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index bcc788147..652401cd3 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -54,6 +54,8 @@ export async function mapController(
});
let result: MapResult;
+ let timeoutHandle: NodeJS.Timeout | null = null;
+
const abort = new AbortController();
try {
result = (await Promise.race([
@@ -78,11 +80,12 @@ export async function mapController(
}),
...(req.body.timeout !== undefined
? [
- new Promise((resolve, reject) =>
- setTimeout(() => {
- abort.abort(new MapTimeoutError());
- reject(new MapTimeoutError());
- }, req.body.timeout),
+ new Promise(
+ (_resolve, reject) =>
+ (timeoutHandle = setTimeout(() => {
+ abort.abort(new MapTimeoutError());
+ reject(new MapTimeoutError());
+ }, req.body.timeout)),
),
]
: []),
@@ -97,6 +100,10 @@ export async function mapController(
} else {
throw error;
}
+ } finally {
+ if (timeoutHandle) {
+ clearTimeout(timeoutHandle);
+ }
}
// Bill the team
@@ -142,6 +149,8 @@ export async function mapController(
num_tokens: 0,
credits_billed: 1,
zeroDataRetention: false, // not supported
+ }).catch(error => {
+ logger.error(`Failed to log job for team ${req.auth.team_id}: ${error}`);
});
// Log final timing information
diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.ts
index a1064b4a7..5e0252025 100644
--- a/apps/api/src/controllers/v2/scrape-status.ts
+++ b/apps/api/src/controllers/v2/scrape-status.ts
@@ -3,6 +3,15 @@ import { getJob } from "./crawl-status";
import { logger as _logger } from "../../lib/logger";
export async function scrapeStatusController(req: any, res: any) {
+ const uuidReg =
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+ if (!req.params.jobId || !uuidReg.test(req.params.jobId)) {
+ return res.status(400).json({
+ success: false,
+ error: "Invalid crawl ID",
+ });
+ }
+
const logger = _logger.child({
module: "scrape-status",
method: "scrapeStatusController",
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 7d3bd08c7..bbc80c1ca 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -8,13 +8,15 @@ import {
ScrapeResponse,
} from "./types";
import { v4 as uuidv4 } from "uuid";
-import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
-import { getJobPriority } from "../../lib/job-priority";
import { hasFormatOfType } from "../../lib/format-utils";
import { TransportableError } from "../../lib/error";
-import { scrapeQueue } from "../../services/worker/nuq";
+import { NuQJob } from "../../services/worker/nuq";
import { checkPermissions } from "../../lib/permissions";
import { withSpan, setSpanAttributes, SpanKind } from "../../lib/otel-tracer";
+import { processJobInternal } from "../../services/worker/scrape-worker";
+import { ScrapeJobData } from "../../types";
+import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";
+import { getJobPriority } from "../../lib/job-priority";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -78,6 +80,7 @@ export async function scrapeController(
const logger = _logger.child({
method: "scrapeController",
jobId,
+ noq: true,
scrapeId: jobId,
teamId: req.auth.team_id,
team_id: req.auth.team_id,
@@ -107,59 +110,6 @@ export async function scrapeController(
process.env.SEARCH_PREVIEW_TOKEN !== undefined &&
process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
- const jobPriority = await getJobPriority({
- team_id: req.auth.team_id,
- basePriority: 10,
- });
-
- // Job enqueuing span
- const job = await withSpan(
- "api.scrape.enqueue_job",
- async enqueueSpan => {
- const result = await addScrapeJob(
- {
- url: req.body.url,
- mode: "single_urls",
- team_id: req.auth.team_id,
- scrapeOptions: {
- ...req.body,
- ...(req.body.__experimental_cache
- ? {
- maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
- }
- : {}),
- },
- internalOptions: {
- teamId: req.auth.team_id,
- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME
- ? true
- : false,
- unnormalizedSourceURL: preNormalizedBody.url,
- bypassBilling: isDirectToBullMQ,
- zeroDataRetention,
- teamFlags: req.acuc?.flags ?? null,
- },
- origin,
- integration: req.body.integration,
- startTime: controllerStartTime,
- zeroDataRetention,
- apiKeyId: req.acuc?.api_key_id ?? null,
- },
- jobId,
- jobPriority,
- isDirectToBullMQ,
- true,
- );
-
- setSpanAttributes(enqueueSpan, {
- "job.priority": jobPriority,
- "job.direct_to_bullmq": isDirectToBullMQ,
- });
-
- return result;
- },
- );
-
const totalWait =
(req.body.waitFor ?? 0) +
(req.body.actions ?? []).reduce(
@@ -167,33 +117,110 @@ export async function scrapeController(
0,
);
- let doc: Document;
+ let doc: Document | null = null;
+ let timeoutHandle: NodeJS.Timeout | null = null;
try {
- // Wait for job completion span
- doc = await withSpan("api.scrape.wait_for_job", async waitSpan => {
- setSpanAttributes(waitSpan, {
- "wait.timeout": timeout !== undefined ? timeout + totalWait : null,
- "wait.job_id": jobId,
- });
+ const lockStart = Date.now();
+ const aborter = new AbortController();
+ if (timeout) {
+ // Semaphore has 2/3 of the timeout time to get a lock to allow for scrape time
+ timeoutHandle = setTimeout(() => {
+ aborter.abort();
+ }, timeout * 0.667);
+ }
+ req.on("close", () => aborter.abort());
+
+ doc = await teamConcurrencySemaphore.withSemaphore(
+ req.auth.team_id,
+ jobId,
+ req.acuc?.concurrency || 1,
+ aborter.signal,
+ timeout ?? 60_000,
+ async () => {
+ const jobPriority = await getJobPriority({
+ team_id: req.auth.team_id,
+ basePriority: 10,
+ });
- const result = await waitForJob(
- job ?? jobId,
- timeout !== undefined ? timeout + totalWait : null,
- zeroDataRetention,
- logger,
- );
+ // TODO: send 429 on abort
+ const lockTime = Date.now() - lockStart;
- setSpanAttributes(waitSpan, {
- "wait.success": true,
- });
+ logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {
+ teamId: req.auth.team_id,
+ lockTime,
+ });
- return result;
- });
+ // Wait for job completion span
+ const doc = await withSpan(
+ "api.scrape.wait_for_job",
+ async waitSpan => {
+ setSpanAttributes(waitSpan, {
+ "wait.timeout":
+ timeout !== undefined ? timeout + totalWait : null,
+ "wait.job_id": jobId,
+ });
+
+ const job: NuQJob<ScrapeJobData> = {
+ id: jobId,
+
+ status: "active",
+ createdAt: new Date(),
+ priority: jobPriority,
+ data: {
+ url: req.body.url,
+ mode: "single_urls",
+ team_id: req.auth.team_id,
+ scrapeOptions: {
+ ...req.body,
+ ...(req.body.__experimental_cache
+ ? {
+ maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
+ }
+ : {}),
+ },
+ internalOptions: {
+ teamId: req.auth.team_id,
+ saveScrapeResultToGCS: process.env
+ .GCS_FIRE_ENGINE_BUCKET_NAME
+ ? true
+ : false,
+ unnormalizedSourceURL: preNormalizedBody.url,
+ bypassBilling: isDirectToBullMQ,
+ zeroDataRetention,
+ teamFlags: req.acuc?.flags ?? null,
+ },
+ skipNuq: true,
+ origin,
+ integration: req.body.integration,
+ startTime: controllerStartTime,
+ zeroDataRetention,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ },
+ };
+
+ const result = await processJobInternal(job);
+
+ setSpanAttributes(waitSpan, {
+ "wait.success": true,
+ });
+
+ return result ?? null;
+ },
+ );
+
+ return doc;
+ },
+ );
} catch (e) {
- logger.error(`Error in scrapeController`, {
- version: "v2",
- error: e,
- });
+ const timeoutErr =
+ e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";
+
+ if (!timeoutErr) {
+ logger.error(`Error in scrapeController`, {
+ version: "v2",
+ error: e,
+ });
+ }
setSpanAttributes(span, {
"scrape.error": e instanceof Error ? e.message : String(e),
@@ -201,10 +228,6 @@ export async function scrapeController(
e instanceof TransportableError ? e.code : "unknown",
});
- if (zeroDataRetention) {
- await scrapeQueue.removeJob(jobId, logger);
- }
-
if (e instanceof TransportableError) {
// DNS resolution errors should return 200 with success: false
if (e.code === "SCRAPE_DNS_RESOLUTION_ERROR") {
@@ -236,10 +259,12 @@ export async function scrapeController(
error: `(Internal server error) - ${e && e.message ? e.message : e}`,
});
}
+ } finally {
+ if (timeoutHandle) {
+ clearTimeout(timeoutHandle);
+ }
}
- await scrapeQueue.removeJob(jobId, logger);
-
if (!hasFormatOfType(req.body.formats, "rawHtml")) {
if (doc && doc.rawHtml) {
delete doc.rawHtml;
@@ -273,7 +298,7 @@ export async function scrapeController(
return res.status(200).json({
success: true,
- data: doc,
+ data: doc!,
scrape_id: origin?.includes("website") ? jobId : undefined,
});
},
diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.ts
index d3570d362..d59a68968 100644
--- a/apps/api/src/lib/extract/extraction-service.ts
+++ b/apps/api/src/lib/extract/extraction-service.ts
@@ -492,9 +492,9 @@ export async function performExtraction(
ajv.compile(multiEntitySchema);
// Wrap in timeout promise
- const timeoutPromise = new Promise(resolve => {
- setTimeout(() => resolve(null), timeoutCompletion);
- });
+ // const timeoutPromise = new Promise(resolve => {
+ // setTimeout(() => resolve(null), timeoutCompletion);
+ // });
const completionPromise = batchExtractPromise(
{
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index d79e64ce6..a4850d19a 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -439,8 +439,9 @@ export async function performExtraction_F0(
ajv.compile(multiEntitySchema);
// Wrap in timeout promise
+ let timeoutHandle: NodeJS.Timeout;
const timeoutPromise = new Promise(resolve => {
- setTimeout(() => resolve(null), timeoutCompletion);
+ timeoutHandle = setTimeout(() => resolve(null), timeoutCompletion);
});
// Check if page should be extracted before proceeding
@@ -504,7 +505,9 @@ export async function performExtraction_F0(
const multiEntityCompletion = (await Promise.race([
completionPromise,
timeoutPromise,
- ])) as Awaited<ReturnType<typeof generateCompletions_F0>>;
+ ]).finally(() => {
+ clearTimeout(timeoutHandle);
+ })) as Awaited<ReturnType<typeof generateCompletions_F0>>;
// Track multi-entity extraction tokens
if (multiEntityCompletion) {
diff --git a/apps/api/src/lib/extract/fire-0/reranker-f0.ts b/apps/api/src/lib/extract/fire-0/reranker-f0.ts
index 58e33e8fd..c295d7f77 100644
--- a/apps/api/src/lib/extract/fire-0/reranker-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/reranker-f0.ts
@@ -78,8 +78,9 @@ export async function rerankLinksWithLLM_F0(
for (let retry = 0; retry <= MAX_RETRIES; retry++) {
try {
+ let timeoutHandle: NodeJS.Timeout;
const timeoutPromise = new Promise<null>(resolve => {
- setTimeout(() => resolve(null), TIMEOUT_MS);
+ timeoutHandle = setTimeout(() => resolve(null), TIMEOUT_MS);
});
// dumpToFile(new Date().toISOString(),[buildRerankerSystemPrompt(), buildRerankerUserPrompt(searchQuery), schema, linksContent])
@@ -114,7 +115,9 @@ export async function rerankLinksWithLLM_F0(
const completion = await Promise.race([
completionPromise,
timeoutPromise,
- ]);
+ ]).finally(() => {
+ clearTimeout(timeoutHandle);
+ });
if (!completion) {
// console.log(`Chunk ${chunkIndex + 1}: Timeout on attempt ${retry + 1}`);
diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.ts
index 44a1b8b30..092a0d059 100644
--- a/apps/api/src/lib/extract/reranker.ts
+++ b/apps/api/src/lib/extract/reranker.ts
@@ -100,9 +100,9 @@ export async function rerankLinksWithLLM(
for (let retry = 0; retry <= MAX_RETRIES; retry++) {
try {
- const timeoutPromise = new Promise<null>(resolve => {
- setTimeout(() => resolve(null), TIMEOUT_MS);
- });
+ // const timeoutPromise = new Promise<null>(resolve => {
+ // setTimeout(() => resolve(null), TIMEOUT_MS);
+ // });
const systemPrompt = `You are analyzing URLs for ${isMultiEntity ? "collecting multiple items" : "specific information"}.
The user's query is: ${searchQuery}
diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.ts
new file mode 100644
index 000000000..0541f4234
--- /dev/null
+++ b/apps/api/src/lib/html-to-markdown-client.ts
@@ -0,0 +1,113 @@
+/**
+ * HTTP Client for HTML to Markdown conversion service
+ *
+ * This client communicates with the Go-based HTML to Markdown microservice
+ * to avoid blocking Node.js event loop with heavy conversions.
+ */
+
+import axios, { AxiosInstance, AxiosError } from "axios";
+import { logger } from "./logger";
+import * as Sentry from "@sentry/node";
+
+interface ConvertRequest {
+ html: string;
+}
+
+interface ConvertResponse {
+ markdown: string;
+ success: boolean;
+}
+
+interface ErrorResponse {
+ error: string;
+ success: boolean;
+}
+
+/**
+ * Convert HTML to Markdown using direct axios call
+ * @param html HTML string to convert
+ * @param serviceUrl URL of the HTML to Markdown service (default: http://localhost:8080)
+ * @returns Markdown string
+ * @throws Error if conversion fails
+ */
+export async function convertHTMLToMarkdownWithHttpService(
+ html: string,
+ serviceUrl?: string,
+): Promise<string> {
+ if (!html || html.trim() === "") {
+ return "";
+ }
+
+ const url =
+ serviceUrl ||
+ process.env.HTML_TO_MARKDOWN_SERVICE_URL ||
+ "http://localhost:8080";
+ const startTime = Date.now();
+
+ try {
+ const request: ConvertRequest = { html };
+
+ const response = await axios.post<ConvertResponse>(
+ `${url}/convert`,
+ request,
+ {
+ timeout: 30000, // 30 second timeout
+ headers: {
+ "Content-Type": "application/json",
+ },
+ },
+ );
+
+ const duration = Date.now() - startTime;
+
+ if (!response.data.success) {
+ throw new Error("Conversion was not successful");
+ }
+
+ logger.debug("HTML to Markdown conversion successful", {
+ duration_ms: duration,
+ input_size: html.length,
+ output_size: response.data.markdown.length,
+ });
+
+ return response.data.markdown;
+ } catch (error) {
+ const duration = Date.now() - startTime;
+
+ if (axios.isAxiosError(error)) {
+ const axiosError = error as AxiosError<ErrorResponse>;
+
+ const errorMessage =
+ axiosError.response?.data?.error || axiosError.message;
+ const statusCode = axiosError.response?.status;
+
+ logger.error("HTML to Markdown conversion failed", {
+ error: errorMessage,
+ statusCode,
+ duration_ms: duration,
+ serviceUrl: url,
+ });
+
+ // Capture in Sentry with additional context
+ Sentry.captureException(error, {
+ tags: {
+ service: "html-to-markdown",
+ status_code: statusCode,
+ },
+ extra: {
+ serviceUrl: url,
+ errorMessage,
+ inputSize: html.length,
+ },
+ });
+
+ throw new Error(`HTML to Markdown conversion failed: ${errorMessage}`);
+ } else {
+ logger.error("Unexpected error during HTML to Markdown conversion", {
+ error,
+ });
+ Sentry.captureException(error);
+ throw error;
+ }
+ }
+}
diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.ts
index 5a21b9b1f..04b162bda 100644
--- a/apps/api/src/lib/html-to-markdown.ts
+++ b/apps/api/src/lib/html-to-markdown.ts
@@ -6,6 +6,8 @@ import dotenv from "dotenv";
import { logger } from "./logger";
import { stat } from "fs/promises";
import { HTML_TO_MARKDOWN_PATH } from "../natives";
+import { convertHTMLToMarkdownWithHttpService } from "./html-to-markdown-client";
+import { postProcessMarkdown } from "@mendable/firecrawl-rs";
dotenv.config();
// TODO: add a timeout to the Go parser
@@ -57,13 +59,30 @@ export async function parseMarkdown(
return "";
}
+ // Try HTTP service first if enabled
+ if (process.env.HTML_TO_MARKDOWN_SERVICE_URL) {
+ try {
+ let markdownContent = await convertHTMLToMarkdownWithHttpService(html);
+ markdownContent = await postProcessMarkdown(markdownContent);
+ return markdownContent;
+ } catch (error) {
+ logger.error(
+ "Error converting HTML to Markdown with HTTP service, falling back to original parser",
+ { error },
+ );
+ Sentry.captureException(error, {
+ tags: {
+ fallback: "original_parser",
+ },
+ });
+ }
+ }
+
try {
if (process.env.USE_GO_MARKDOWN_PARSER == "true") {
const converter = await GoMarkdownConverter.getInstance();
let markdownContent = await converter.convertHTMLToMarkdown(html);
-
- markdownContent = processMultiLineLinks(markdownContent);
- markdownContent = removeSkipToContentLinks(markdownContent);
+ markdownContent = await postProcessMarkdown(markdownContent);
// logger.info(`HTML to Markdown conversion using Go parser successful`);
return markdownContent;
}
@@ -108,8 +127,7 @@ export async function parseMarkdown(
try {
let markdownContent = await turndownService.turndown(html);
- markdownContent = processMultiLineLinks(markdownContent);
- markdownContent = removeSkipToContentLinks(markdownContent);
+ markdownContent = await postProcessMarkdown(markdownContent);
return markdownContent;
} catch (error) {
diff --git a/apps/api/src/lib/retry-utils.ts b/apps/api/src/lib/retry-utils.ts
index eada3ceec..6246e8a9c 100644
--- a/apps/api/src/lib/retry-utils.ts
+++ b/apps/api/src/lib/retry-utils.ts
@@ -54,7 +54,7 @@ export async function attemptRequest<T>(
* Abortable sleep function that resolves immediately if the signal is aborted
*/
function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {
- return new Promise((resolve, reject) => {
+ return new Promise(resolve => {
if (signal?.aborted) {
resolve();
return;
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index 770624b45..1ddea4ef5 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -49,68 +49,68 @@ v1Router.use(requestTimingMiddleware("v1"));
// Configure payment middleware to enable micropayment-protected endpoints
// This middleware handles payment verification and processing for premium API features
// x402 payments protocol - https://github.com/coinbase/x402
-v1Router.use(
- paymentMiddleware(
- (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
- "0x0000000000000000000000000000000000000000",
- {
- "POST /x402/search": {
- price: process.env.X402_ENDPOINT_PRICE_USD as string,
- network: process.env.X402_NETWORK as
- | "base-sepolia"
- | "base"
- | "avalanche-fuji"
- | "avalanche"
- | "iotex",
- config: {
- discoverable: true,
- description:
- "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",
- mimeType: "application/json",
- maxTimeoutSeconds: 120,
- inputSchema: {
- body: {
- query: {
- type: "string",
- description: "Search query to find relevant web pages",
- required: true,
- },
- limit: {
- type: "number",
- description: "Maximum number of results to return (max 10)",
- required: false,
- },
- scrapeOptions: {
- type: "object",
- description: "Options for scraping the found pages",
- required: false,
- },
- },
- },
- outputSchema: {
- type: "object",
- properties: {
- success: { type: "boolean" },
- data: {
- type: "array",
- items: {
- type: "object",
- properties: {
- url: { type: "string" },
- title: { type: "string" },
- description: { type: "string" },
- markdown: { type: "string" },
- },
- },
- },
- },
- },
- },
- },
- },
- facilitator,
- ),
-);
+// v1Router.use(
+// paymentMiddleware(
+// (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
+// "0x0000000000000000000000000000000000000000",
+// {
+// "POST /x402/search": {
+// price: process.env.X402_ENDPOINT_PRICE_USD as string,
+// network: process.env.X402_NETWORK as
+// | "base-sepolia"
+// | "base"
+// | "avalanche-fuji"
+// | "avalanche"
+// | "iotex",
+// config: {
+// discoverable: true,
+// description:
+// "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",
+// mimeType: "application/json",
+// maxTimeoutSeconds: 120,
+// inputSchema: {
+// body: {
+// query: {
+// type: "string",
+// description: "Search query to find relevant web pages",
+// required: true,
+// },
+// limit: {
+// type: "number",
+// description: "Maximum number of results to return (max 10)",
+// required: false,
+// },
+// scrapeOptions: {
+// type: "object",
+// description: "Options for scraping the found pages",
+// required: false,
+// },
+// },
+// },
+// outputSchema: {
+// type: "object",
+// properties: {
+// success: { type: "boolean" },
+// data: {
+// type: "array",
+// items: {
+// type: "object",
+// properties: {
+// url: { type: "string" },
+// title: { type: "string" },
+// description: { type: "string" },
+// markdown: { type: "string" },
+// },
+// },
+// },
+// },
+// },
+// },
+// },
+// },
+// facilitator,
+// ),
+// );
v1Router.post(
"/scrape",
@@ -310,5 +310,65 @@ v1Router.post(
"/x402/search",
authMiddleware(RateLimiterMode.Search),
countryCheck,
+ paymentMiddleware(
+ (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
+ "0x0000000000000000000000000000000000000000",
+ {
+ "POST /x402/search": {
+ price: process.env.X402_ENDPOINT_PRICE_USD as string,
+ network: process.env.X402_NETWORK as
+ | "base-sepolia"
+ | "base"
+ | "avalanche-fuji"
+ | "avalanche"
+ | "iotex",
+ config: {
+ discoverable: true,
+ description:
+ "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",
+ mimeType: "application/json",
+ maxTimeoutSeconds: 120,
+ inputSchema: {
+ body: {
+ query: {
+ type: "string",
+ description: "Search query to find relevant web pages",
+ required: true,
+ },
+ limit: {
+ type: "number",
+ description: "Maximum number of results to return (max 10)",
+ required: false,
+ },
+ scrapeOptions: {
+ type: "object",
+ description: "Options for scraping the found pages",
+ required: false,
+ },
+ },
+ },
+ outputSchema: {
+ type: "object",
+ properties: {
+ success: { type: "boolean" },
+ data: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ url: { type: "string" },
+ title: { type: "string" },
+ description: { type: "string" },
+ markdown: { type: "string" },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ },
+ facilitator,
+ ),
wrap(x402SearchController),
);
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 86b82ef63..87d58d767 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -44,117 +44,117 @@ v2Router.use(requestTimingMiddleware("v2"));
// Configure payment middleware to enable micropayment-protected endpoints
// This middleware handles payment verification and processing for premium API features
// x402 payments protocol - https://github.com/coinbase/x402
-v2Router.use(
- paymentMiddleware(
- (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
- "0x0000000000000000000000000000000000000000",
- {
- "POST /x402/search": {
- price: process.env.X402_ENDPOINT_PRICE_USD as string,
- network: process.env.X402_NETWORK as
- | "base-sepolia"
- | "base"
- | "avalanche-fuji"
- | "avalanche"
- | "iotex",
- config: {
- discoverable: true,
- description:
- "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",
- mimeType: "application/json",
- maxTimeoutSeconds: 120,
- inputSchema: {
- body: {
- query: {
- type: "string",
- description: "Search query to find relevant web pages",
- required: true,
- },
- sources: {
- type: "array",
- description: "Sources to search (web, news, images)",
- required: false,
- },
- limit: {
- type: "number",
- description: "Maximum number of results to return (max 10)",
- required: false,
- },
- scrapeOptions: {
- type: "object",
- description: "Options for scraping the found pages",
- required: false,
- },
- asyncScraping: {
- type: "boolean",
- description: "Whether to return job IDs for async scraping",
- required: false,
- },
- },
- },
- outputSchema: {
- type: "object",
- properties: {
- success: { type: "boolean" },
- data: {
- type: "object",
- properties: {
- web: {
- type: "array",
- items: {
- type: "object",
- properties: {
- url: { type: "string" },
- title: { type: "string" },
- description: { type: "string" },
- markdown: { type: "string" },
- },
- },
- },
- news: {
- type: "array",
- items: {
- type: "object",
- properties: {
- url: { type: "string" },
- title: { type: "string" },
- snippet: { type: "string" },
- markdown: { type: "string" },
- },
- },
- },
- images: {
- type: "array",
- items: {
- type: "object",
- properties: {
- url: { type: "string" },
- title: { type: "string" },
- markdown: { type: "string" },
- },
- },
- },
- },
- },
- scrapeIds: {
- type: "object",
- description:
- "Job IDs for async scraping (if asyncScraping is true)",
- properties: {
- web: { type: "array", items: { type: "string" } },
- news: { type: "array", items: { type: "string" } },
- images: { type: "array", items: { type: "string" } },
- },
- },
- creditsUsed: { type: "number" },
- },
- },
- },
- },
- },
- facilitator,
- ),
-);
+// v2Router.use(
+// paymentMiddleware(
+// (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
+// "0x0000000000000000000000000000000000000000",
+// {
+// "POST /x402/search": {
+// price: process.env.X402_ENDPOINT_PRICE_USD as string,
+// network: process.env.X402_NETWORK as
+// | "base-sepolia"
+// | "base"
+// | "avalanche-fuji"
+// | "avalanche"
+// | "iotex",
+// config: {
+// discoverable: true,
+// description:
+// "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",
+// mimeType: "application/json",
+// maxTimeoutSeconds: 120,
+// inputSchema: {
+// body: {
+// query: {
+// type: "string",
+// description: "Search query to find relevant web pages",
+// required: true,
+// },
+// sources: {
+// type: "array",
+// description: "Sources to search (web, news, images)",
+// required: false,
+// },
+// limit: {
+// type: "number",
+// description: "Maximum number of results to return (max 10)",
+// required: false,
+// },
+// scrapeOptions: {
+// type: "object",
+// description: "Options for scraping the found pages",
+// required: false,
+// },
+// asyncScraping: {
+// type: "boolean",
+// description: "Whether to return job IDs for async scraping",
+// required: false,
+// },
+// },
+// },
+// outputSchema: {
+// type: "object",
+// properties: {
+// success: { type: "boolean" },
+// data: {
+// type: "object",
+// properties: {
+// web: {
+// type: "array",
+// items: {
+// type: "object",
+// properties: {
+// url: { type: "string" },
+// title: { type: "string" },
+// description: { type: "string" },
+// markdown: { type: "string" },
+// },
+// },
+// },
+// news: {
+// type: "array",
+// items: {
+// type: "object",
+// properties: {
+// url: { type: "string" },
+// title: { type: "string" },
+// snippet: { type: "string" },
+// markdown: { type: "string" },
+// },
+// },
+// },
+// images: {
+// type: "array",
+// items: {
+// type: "object",
+// properties: {
+// url: { type: "string" },
+// title: { type: "string" },
+// markdown: { type: "string" },
+// },
+// },
+// },
+// },
+// },
+// scrapeIds: {
+// type: "object",
+// description:
+// "Job IDs for async scraping (if asyncScraping is true)",
+// properties: {
+// web: { type: "array", items: { type: "string" } },
+// news: { type: "array", items: { type: "string" } },
+// images: { type: "array", items: { type: "string" } },
+// },
+// },
+// creditsUsed: { type: "number" },
+// },
+// },
+// },
+// },
+// },
+// facilitator,
+// ),
+// );
v2Router.post(
"/search",
@@ -314,5 +314,114 @@ v2Router.post(
authMiddleware(RateLimiterMode.Search),
countryCheck,
blocklistMiddleware,
+ paymentMiddleware(
+ (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||
+ "0x0000000000000000000000000000000000000000",
+ {
+ "POST /x402/search": {
+ price: process.env.X402_ENDPOINT_PRICE_USD as string,
+ network: process.env.X402_NETWORK as
+ | "base-sepolia"
+ | "base"
+ | "avalanche-fuji"
+ | "avalanche"
+ | "iotex",
+ config: {
+ discoverable: true,
+ description:
+ "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",
+ mimeType: "application/json",
+ maxTimeoutSeconds: 120,
+ inputSchema: {
+ body: {
+ query: {
+ type: "string",
+ description: "Search query to find relevant web pages",
+ required: true,
+ },
+ sources: {
+ type: "array",
+ description: "Sources to search (web, news, images)",
+ required: false,
+ },
+ limit: {
+ type: "number",
+ description: "Maximum number of results to return (max 10)",
+ required: false,
+ },
+ scrapeOptions: {
+ type: "object",
+ description: "Options for scraping the found pages",
+ required: false,
+ },
+ asyncScraping: {
+ type: "boolean",
+ description: "Whether to return job IDs for async scraping",
+ required: false,
+ },
+ },
+ },
+ outputSchema: {
+ type: "object",
+ properties: {
+ success: { type: "boolean" },
+ data: {
+ type: "object",
+ properties: {
+ web: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ url: { type: "string" },
+ title: { type: "string" },
+ description: { type: "string" },
+ markdown: { type: "string" },
+ },
+ },
+ },
+ news: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ url: { type: "string" },
+ title: { type: "string" },
+ snippet: { type: "string" },
+ markdown: { type: "string" },
+ },
+ },
+ },
+ images: {
+ type: "array",
+ items: {
+ type: "object",
+ properties: {
+ url: { type: "string" },
+ title: { type: "string" },
+ markdown: { type: "string" },
+ },
+ },
+ },
+ },
+ },
+ scrapeIds: {
+ type: "object",
+ description:
+ "Job IDs for async scraping (if asyncScraping is true)",
+ properties: {
+ web: { type: "array", items: { type: "string" } },
+ news: { type: "array", items: { type: "string" } },
+ images: { type: "array", items: { type: "string" } },
+ },
+ },
+ creditsUsed: { type: "number" },
+ },
+ },
+ },
+ },
+ },
+ facilitator,
+ ),
wrap(x402SearchController),
);
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 228ef32b8..6f5371ddd 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -483,8 +483,12 @@ export class WebCrawler {
}
};
+ let timeoutHandle: NodeJS.Timeout;
const timeoutPromise = new Promise((_, reject) => {
- setTimeout(() => reject(new Error("Sitemap fetch timeout")), timeout);
+ timeoutHandle = setTimeout(
+ () => reject(new Error("Sitemap fetch timeout")),
+ timeout,
+ );
});
// Allow sitemaps to be cached for 48 hours if they are requested from /map
@@ -515,7 +519,9 @@ export class WebCrawler {
),
]).then(results => results.reduce((a, x) => a + x, 0)),
timeoutPromise,
- ])) as number;
+ ]).finally(() => {
+ clearTimeout(timeoutHandle);
+ })) as number;
if (count > 0) {
if (
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 185e1c64c..a7f37f4a9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -260,6 +260,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
} else {
const file = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {
headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
});
const ct = file.response.headers.get("Content-Type");
@@ -295,6 +296,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }
: await downloadFile(meta.id, meta.rewrittenUrl ?? meta.url, {
headers: meta.options.headers,
+ signal: meta.abort.asSignal(),
});
if ((response as any).headers) {
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
index 831f40d90..21b14338b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts
@@ -48,15 +48,19 @@ export async function downloadFile(
throw new EngineError("Response body was null", { cause: { response } });
}
- response.body.pipeTo(Writable.toWeb(tempFileWrite));
- await new Promise((resolve, reject) => {
- tempFileWrite.on("finish", () => resolve(null));
- tempFileWrite.on("error", error => {
- reject(
- new EngineError("Failed to write to temp file", { cause: { error } }),
- );
- });
- });
+ try {
+ await response.body
+ .pipeTo(Writable.toWeb(tempFileWrite), {
+ signal: init?.signal || undefined,
+ })
+ .catch(error => {
+ throw new EngineError("Failed to write to temp file", {
+ cause: { error },
+ });
+ });
+ } finally {
+ tempFileWrite.close();
+ }
return {
response,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 2267f3b39..ad05a836f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -4,6 +4,7 @@ import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
import {
type Document,
+ getPDFMaxPages,
scrapeOptions,
type ScrapeOptions,
type TeamFlags,
@@ -114,6 +115,7 @@ export type Meta = {
| undefined; // undefined: no prefetch yet, null: prefetch came back empty
costTracking: CostTracking;
winnerEngine?: Engine;
+ abortHandle?: NodeJS.Timeout;
};
function buildFeatureFlags(
@@ -278,6 +280,18 @@ async function buildMetaObject(
crawlId: internalOptions.crawlId,
});
+ const abortController = new AbortController();
+ const abortHandle =
+ options.timeout !== undefined
+ ? setTimeout(
+ () =>
+ abortController.abort(
+ new ScrapeJobTimeoutError("Scrape timed out"),
+ ),
+ options.timeout,
+ )
+ : undefined;
+
return {
id,
url,
@@ -293,11 +307,12 @@ async function buildMetaObject(
},
internalOptions,
logger,
+ abortHandle,
abort: new AbortManager(
internalOptions.externalAbort,
options.timeout !== undefined
? {
- signal: AbortSignal.timeout(options.timeout),
+ signal: abortController.signal,
tier: "scrape",
timesOutAt: new Date(Date.now() + options.timeout),
throwable() {
@@ -357,63 +372,79 @@ async function scrapeURLLoopIter(
engine: Engine,
snipeAbort,
): Promise<EngineScrapeResult> {
- const engineResult = await scrapeURLWithEngine(
- {
- ...meta,
- abort: meta.abort.child(snipeAbort),
- },
- engine,
- );
-
- let checkMarkdown = await parseMarkdown(
- await htmlTransform(
- engineResult.html,
- meta.url,
- scrapeOptions.parse({ onlyMainContent: true }),
- ),
- );
-
- if (checkMarkdown.trim().length === 0) {
- checkMarkdown = await parseMarkdown(
- await htmlTransform(
- engineResult.html,
- meta.url,
- scrapeOptions.parse({ onlyMainContent: false }),
- ),
+ const abort = meta.abort.child(snipeAbort);
+ try {
+ const engineResult = await scrapeURLWithEngine(
+ {
+ ...meta,
+ abort,
+ },
+ engine,
);
- }
- // Success factors
- const isLongEnough = checkMarkdown.trim().length > 0;
- const isGoodStatusCode =
- (engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||
- engineResult.statusCode === 304;
- const hasNoPageError = engineResult.error === undefined;
- const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode);
+ let checkMarkdown: string;
+ if (meta.internalOptions.teamId === "sitemap") {
+ checkMarkdown = engineResult.html?.trim() ?? "";
+ } else {
+ checkMarkdown = await parseMarkdown(
+ await htmlTransform(
+ engineResult.html,
+ meta.url,
+ scrapeOptions.parse({ onlyMainContent: true }),
+ ),
+ );
+
+ if (checkMarkdown.trim().length === 0) {
+ checkMarkdown = await parseMarkdown(
+ await htmlTransform(
+ engineResult.html,
+ meta.url,
+ scrapeOptions.parse({ onlyMainContent: false }),
+ ),
+ );
+ }
+ }
- if (
- isLikelyProxyError &&
- meta.options.proxy === "auto" &&
- !meta.featureFlags.has("stealthProxy")
- ) {
- meta.logger.info(
- "Scrape via " +
- engine +
- " deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.",
+ // Success factors
+ const isLongEnough = checkMarkdown.trim().length > 0;
+ const isGoodStatusCode =
+ (engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||
+ engineResult.statusCode === 304;
+ const hasNoPageError = engineResult.error === undefined;
+ const isLikelyProxyError = [401, 403, 429].includes(
+ engineResult.statusCode,
);
- throw new AddFeatureError(["stealthProxy"]);
- }
- // NOTE: TODO: what to do when status code is bad is tough...
- // we cannot just rely on text because error messages can be brief and not hit the limit
- // should we just use all the fallbacks and pick the one with the longest text? - mogery
- if (isLongEnough || !isGoodStatusCode) {
- meta.logger.info("Scrape via " + engine + " deemed successful.", {
- factors: { isLongEnough, isGoodStatusCode, hasNoPageError },
- });
- return engineResult;
- } else {
- throw new EngineUnsuccessfulError(engine);
+ if (
+ isLikelyProxyError &&
+ meta.options.proxy === "auto" &&
+ !meta.featureFlags.has("stealthProxy")
+ ) {
+ meta.logger.info(
+ "Scrape via " +
+ engine +
+ " deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.",
+ );
+ throw new AddFeatureError(["stealthProxy"]);
+ }
+
+ // NOTE: TODO: what to do when status code is bad is tough...
+ // we cannot just rely on text because error messages can be brief and not hit the limit
+ // should we just use all the fallbacks and pick the one with the longest text? - mogery
+ if (isLongEnough || !isGoodStatusCode) {
+ meta.logger.info("Scrape via " + engine + " deemed successful.", {
+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },
+ });
+ return engineResult;
+ } else {
+ meta.logger.warn("Scrape via " + engine + " deemed unsuccessful.", {
+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },
+ length: engineResult.html?.trim().length ?? 0,
+ });
+ throw new EngineUnsuccessfulError(engine);
+ }
+ } finally {
+ abort?.dispose();
}
}
@@ -539,35 +570,40 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
});
while (true) {
+ let timeouts: NodeJS.Timeout[] = [];
try {
result = await Promise.race([
...enginePromises.map(x => x.promise),
...(remainingEngines.length > 0
? [
new Promise<EngineScrapeResultWithContext>((_, reject) => {
- setTimeout(() => {
- reject(new WaterfallNextEngineSignal());
- }, waitUntilWaterfall);
+ timeouts.push(
+ setTimeout(() => {
+ reject(new WaterfallNextEngineSignal());
+ }, waitUntilWaterfall),
+ );
}),
]
: []),
new Promise<EngineScrapeResultWithContext>((_, reject) => {
- setTimeout(() => {
- try {
- meta.abort.throwIfAborted();
-
- // Fallback error if above doesn't throw
- const usingDefaultTimeout =
- meta.abort.scrapeTimeout() === undefined;
- throw new ScrapeJobTimeoutError(
- usingDefaultTimeout
- ? "Scrape timed out due to maximum length of 5 minutes"
- : "Scrape timed out",
- );
- } catch (error) {
- reject(error);
- }
- }, meta.abort.scrapeTimeout() ?? 300000);
+ timeouts.push(
+ setTimeout(() => {
+ try {
+ meta.abort.throwIfAborted();
+
+ // Fallback error if above doesn't throw
+ const usingDefaultTimeout =
+ meta.abort.scrapeTimeout() === undefined;
+ throw new ScrapeJobTimeoutError(
+ usingDefaultTimeout
+ ? "Scrape timed out due to maximum length of 5 minutes"
+ : "Scrape timed out",
+ );
+ } catch (error) {
+ reject(error);
+ }
+ }, meta.abort.scrapeTimeout() ?? 300000),
+ );
}),
]);
break;
@@ -650,6 +686,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
} else if (error instanceof WaterfallNextEngineSignal) {
// It's time to waterfall the next engine
break;
+ } else if (error instanceof ScrapeJobTimeoutError) {
+ throw error;
} else if (error instanceof AbortManagerThrownError) {
if (error.tier === "engine") {
meta.logger.warn(
@@ -665,6 +703,10 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
});
throw error;
}
+ } finally {
+ for (const to of timeouts) {
+ clearTimeout(to);
+ }
}
}
@@ -775,6 +817,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
: warning;
}
+ // NOTE: for sitemap, we don't need all the transformers, need to skip unused ones
document = await executeTransformers(meta, document);
// Set final span attributes
@@ -1075,6 +1118,28 @@ export async function scrapeURL(
result.success && result.document.metadata.cacheState === "hit",
});
+ if (useIndex) {
+ meta.logger.debug("scrapeURL index metrics", {
+ module: "scrapeURL/index-metrics",
+ timeTaken: Date.now() - startTime,
+ changeTracking: hasFormatOfType(
+ meta.options.formats,
+ "changeTracking",
+ ),
+ branding: hasFormatOfType(meta.options.formats, "branding"),
+ pdfMaxPages: getPDFMaxPages(meta.options.parsers),
+ maxAge: meta.options.maxAge,
+ headers: meta.options.headers
+ ? Object.keys(meta.options.headers).length
+ : 0,
+ actions: meta.options.actions?.length ?? 0,
+ proxy: meta.options.proxy,
+ success: result.success,
+ indexHit:
+ result.success && result.document.metadata.cacheState === "hit",
+ });
+ }
+
setSpanAttributes(span, {
"scrape.success": true,
"scrape.duration_ms": Date.now() - startTime,
@@ -1096,6 +1161,27 @@ export async function scrapeURL(
indexHit: false,
});
+ if (useIndex) {
+ meta.logger.debug("scrapeURL index metrics", {
+ module: "scrapeURL/index-metrics",
+ timeTaken: Date.now() - startTime,
+ changeTracking: hasFormatOfType(
+ meta.options.formats,
+ "changeTracking",
+ ),
+ branding: hasFormatOfType(meta.options.formats, "branding"),
+ pdfMaxPages: getPDFMaxPages(meta.options.parsers),
+ maxAge: meta.options.maxAge,
+ headers: meta.options.headers
+ ? Object.keys(meta.options.headers).length
+ : 0,
+ actions: meta.options.actions?.length ?? 0,
+ proxy: meta.options.proxy,
+ success: false,
+ indexHit: false,
+ });
+ }
+
// Set error attributes on span
let errorType = "unknown";
if (error instanceof NoEnginesLeftError) {
diff --git a/apps/api/src/scraper/scrapeURL/lib/abortManager.ts b/apps/api/src/scraper/scrapeURL/lib/abortManager.ts
index be25f0711..70f7e275b 100644
--- a/apps/api/src/scraper/scrapeURL/lib/abortManager.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/abortManager.ts
@@ -8,6 +8,7 @@ export type AbortInstance = {
export class AbortManager {
private aborts: AbortInstance[] = [];
private mappedController: AbortController | null = null;
+ private listeners: Array<{ signal: AbortSignal; handler: () => void }> = [];
constructor(...instances: (AbortInstance | undefined | null)[]) {
this.aborts = instances.filter(
@@ -15,6 +16,38 @@ export class AbortManager {
) as AbortInstance[];
}
+ dispose() {
+ for (const { signal, handler } of this.listeners) {
+ signal.removeEventListener("abort", handler);
+ }
+
+ this.listeners = [];
+ this.aborts = [];
+ this.mappedController = null;
+ }
+
+ private resolveInner(abort: AbortInstance): any {
+ try {
+ return abort.throwable();
+ } catch (err) {
+ return err;
+ }
+ }
+
+ private register(abort: AbortInstance) {
+ const handler = () => {
+ if (!this.mappedController) return;
+
+ const inner = this.resolveInner(abort);
+ const reason = new AbortManagerThrownError(abort.tier, inner);
+
+ this.mappedController.abort(reason);
+ };
+
+ abort.signal.addEventListener("abort", handler);
+ this.listeners.push({ signal: abort.signal, handler });
+ }
+
add(...instances: (AbortInstance | undefined | null)[]) {
const pureInstances = instances.filter(
x => x !== undefined && x !== null,
@@ -23,11 +56,7 @@ export class AbortManager {
if (this.mappedController !== null) {
for (const abort of pureInstances) {
- abort.signal.addEventListener("abort", () => {
- this.mappedController?.abort(
- new AbortManagerThrownError(abort.tier, abort.throwable()),
- );
- });
+ this.register(abort);
}
}
}
@@ -49,7 +78,8 @@ export class AbortManager {
throwIfAborted(): void {
for (const abort of this.aborts) {
if (abort.signal.aborted) {
- throw new AbortManagerThrownError(abort.tier, abort.throwable());
+ const inner = this.resolveInner(abort);
+ throw new AbortManagerThrownError(abort.tier, inner);
}
}
}
@@ -58,11 +88,7 @@ export class AbortManager {
this.mappedController = new AbortController();
for (const abort of this.aborts) {
- abort.signal.addEventListener("abort", () => {
- this.mappedController?.abort(
- new AbortManagerThrownError(abort.tier, abort.throwable()),
- );
- });
+ this.register(abort);
}
}
diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
index da9866a05..6d87b0ffc 100644
--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts
@@ -220,9 +220,16 @@ export async function robustFetch<
},
);
if (tryCooldown !== undefined) {
- await new Promise(resolve =>
- setTimeout(() => resolve(null), tryCooldown),
- );
+ let timeoutHandle: NodeJS.Timeout | null = null;
+ try {
+ await new Promise<null>(resolve => {
+ timeoutHandle = setTimeout(() => resolve(null), tryCooldown);
+ });
+ } finally {
+ if (timeoutHandle) {
+ clearTimeout(timeoutHandle);
+ }
+ }
}
return await robustFetch({
...params,
diff --git a/apps/api/src/search/v2/ddgsearch.ts b/apps/api/src/search/v2/ddgsearch.ts
index 2200950c6..2f05cbb14 100644
--- a/apps/api/src/search/v2/ddgsearch.ts
+++ b/apps/api/src/search/v2/ddgsearch.ts
@@ -134,7 +134,12 @@ export async function ddgSearch(
let antiBotRetries = 0;
while (results.length < num_results && nextPageData) {
- const timeoutSignal = AbortSignal.timeout(timeout);
+ const abortController = new AbortController();
+ const timeoutHandle = setTimeout(() => {
+ if (abortController) {
+ abortController.abort();
+ }
+ }, timeout);
try {
let response: undici.Response;
@@ -153,7 +158,7 @@ export async function ddgSearch(
"Accept-Encoding": "gzip, deflate, br",
"Upgrade-Insecure-Requests": "1",
},
- signal: timeoutSignal,
+ signal: abortController.signal,
},
);
} else {
@@ -170,7 +175,7 @@ export async function ddgSearch(
"Accept-Encoding": "gzip, deflate, br",
"Upgrade-Insecure-Requests": "1",
},
- signal: timeoutSignal,
+ signal: abortController.signal,
});
}
@@ -204,7 +209,10 @@ export async function ddgSearch(
} else {
throw error;
}
+ } finally {
+ if (timeoutHandle) clearTimeout(timeoutHandle);
}
+
await new Promise(r => setTimeout(r, 1000));
}
diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.ts
index 97b252629..9be7a6057 100644
--- a/apps/api/src/services/ab-test.ts
+++ b/apps/api/src/services/ab-test.ts
@@ -28,15 +28,21 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {
webScraperOptions.internalOptions?.v1Agent === undefined &&
webScraperOptions.internalOptions?.v1JSONAgent === undefined;
if (shouldABTest) {
+ let timeout = Math.min(
+ 60000,
+ (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000,
+ );
+
(async () => {
+ const abortController = new AbortController();
+ const timeoutHandle = setTimeout(() => {
+ if (abortController) {
+ abortController.abort();
+ }
+ }, timeout);
+
try {
abLogger.info("A/B-testing scrapeURL to staging");
- const abort = AbortSignal.timeout(
- Math.min(
- 60000,
- (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000,
- ),
- );
await robustFetch({
url: `http://${abHostEnv}/v2/scrape`,
method: "POST",
@@ -50,11 +56,13 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {
tryCount: 1,
ignoreResponse: true,
mock: null,
- abort,
+ abort: abortController.signal,
});
abLogger.info("A/B-testing scrapeURL (staging) request sent");
} catch (error) {
abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
+ } finally {
+ if (timeoutHandle) clearTimeout(timeoutHandle);
}
})();
}
@@ -84,12 +92,18 @@ export function abTestFireEngine(
Math.random() <= abRate &&
abHostEnv;
if (shouldABTest) {
+ let timeout = Math.min(60000, (feRequest.timeout ?? 30000) + 10000);
+
(async () => {
+ const abortController = new AbortController();
+ const timeoutHandle = setTimeout(() => {
+ if (abortController) {
+ abortController.abort();
+ }
+ }, timeout);
+
try {
abLogger.info("A/B-testing scrapeURL to staging");
- const abort = AbortSignal.timeout(
- Math.min(60000, (feRequest.timeout ?? 30000) + 10000),
- );
await robustFetch({
url: `http://${abHostEnv}/scrape`,
method: "POST",
@@ -98,11 +112,13 @@ export function abTestFireEngine(
tryCount: 1,
ignoreResponse: true,
mock: null,
- abort,
+ abort: abortController.signal,
});
abLogger.info("A/B-testing scrapeURL (staging) request sent");
} catch (error) {
abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });
+ } finally {
+ if (timeoutHandle) clearTimeout(timeoutHandle);
}
})();
}
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index dde434a34..6a009c15b 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -523,6 +523,7 @@ export async function waitForJob(
const jobId = typeof job == "string" ? job : job.id;
const isConcurrencyLimited = !!(typeof job === "string");
+ let timeoutHandle: NodeJS.Timeout | null = null;
let doc: Document | null = null;
try {
doc = await Promise.race(
@@ -534,7 +535,7 @@ export async function waitForJob(
),
timeout !== null
? new Promise<Document>((_resolve, reject) => {
- setTimeout(() => {
+ timeoutHandle = setTimeout(() => {
reject(
new ScrapeJobTimeoutError(
"Scrape timed out" +
@@ -561,7 +562,12 @@ export async function waitForJob(
} else {
throw e;
}
+ } finally {
+ if (timeoutHandle) {
+ clearTimeout(timeoutHandle);
+ }
}
+
logger.debug("Got job");
if (!doc) {
diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.ts
index 5eb11d1af..62a08baf1 100644
--- a/apps/api/src/services/supabase.ts
+++ b/apps/api/src/services/supabase.ts
@@ -8,6 +8,8 @@ class SupabaseService {
private client: SupabaseClient | null = null;
private rrClient: SupabaseClient | null = null;
+ private acucClient: SupabaseClient | null = null;
+
constructor() {
const supabaseUrl = process.env.SUPABASE_URL;
const supabaseReplicaUrl = process.env.SUPABASE_REPLICA_URL;
@@ -15,6 +17,26 @@ class SupabaseService {
const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
// Only initialize the Supabase client if both URL and Service Token are provided.
if (!useDbAuthentication) {
+ if (!!process.env.SUPABASE_ACUC_URL) {
+ const supabaseAcucUrl = process.env.SUPABASE_ACUC_URL;
+ const supabaseAcucServiceToken =
+ process.env.SUPABASE_ACUC_SERVICE_TOKEN;
+ if (!supabaseAcucUrl || !supabaseAcucServiceToken) {
+ logger.error(
+ "Supabase ACUC environment variables aren't configured correctly. Supabase ACUC client will not be initialized. Fix ENV configuration or disable ACUC with USE_DB_AUTHENTICATION env variable",
+ );
+ this.acucClient = null;
+ } else {
+ this.acucClient = createClient(
+ supabaseAcucUrl,
+ supabaseAcucServiceToken,
+ );
+ logger.info(
+ "Supabase ACUC only client initialized, this is only for request authentication",
+ );
+ }
+ }
+
// Warn the user that Authentication is disabled by setting the client to null
logger.warn(
"Authentication is disabled. Supabase client will not be initialized.",
@@ -45,6 +67,10 @@ class SupabaseService {
getRRClient(): SupabaseClient | null {
return this.rrClient;
}
+
+ getACUCOnlyClient(): SupabaseClient | null {
+ return this.acucClient;
+ }
}
const serv = new SupabaseService();
@@ -86,3 +112,21 @@ export const supabase_rr_service: SupabaseClient = new Proxy(serv, {
return Reflect.get(client, prop, receiver);
},
}) as unknown as SupabaseClient;
+
+export const supabase_acuc_only_service: SupabaseClient = new Proxy(serv, {
+ get: function (target, prop, receiver) {
+ const client = target.getACUCOnlyClient();
+ // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.
+ if (client === null) {
+ return () => {
+ throw new Error("Supabase RR client is not configured.");
+ };
+ }
+ // Direct access to SupabaseService properties takes precedence.
+ if (prop in target) {
+ return Reflect.get(target, prop, receiver);
+ }
+ // Otherwise, delegate access to the Supabase client.
+ return Reflect.get(client, prop, receiver);
+ },
+}) as unknown as SupabaseClient;
diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.ts
index 4f89dd362..2f6b23662 100644
--- a/apps/api/src/services/webhook/delivery.ts
+++ b/apps/api/src/services/webhook/delivery.ts
@@ -98,13 +98,23 @@ export class WebhookSender {
headers["X-Firecrawl-Signature"] = `sha256=${hmac.digest("hex")}`;
}
+ const abortController = new AbortController();
+ const timeoutHandle = setTimeout(
+ () => {
+ if (abortController) {
+ abortController.abort();
+ }
+ },
+ this.context.v0 ? 30000 : 10000,
+ );
+
try {
const res = await undici.fetch(this.config.url, {
method: "POST",
headers,
body: payloadString,
dispatcher: getSecureDispatcher(),
- signal: AbortSignal.timeout(this.context.v0 ? 30000 : 10000),
+ signal: abortController.signal,
});
if (!res.ok) {
@@ -146,6 +156,8 @@ export class WebhookSender {
});
throw error;
+ } finally {
+ if (timeoutHandle) clearTimeout(timeoutHandle);
}
}
diff --git a/apps/api/src/services/worker/redis.ts b/apps/api/src/services/worker/redis.ts
new file mode 100644
index 000000000..bca1aa2a8
--- /dev/null
+++ b/apps/api/src/services/worker/redis.ts
@@ -0,0 +1,118 @@
+import IORedis, { Redis } from "ioredis";
+import { logger as _logger } from "../../lib/logger";
+
+const logger = _logger.child({ module: "nuq/redis" });
+
+const luaScripts = {
+ semaphore: {
+ acquire: `
+-- KEYS[1]=leases_zset ; ARGV[1]=holder_id, ARGV[2]=limit, ARGV[3]=lease_ttl_ms
+local t = redis.call('TIME')
+local now_ms = t[1]*1000 + math.floor(t[2]/1000)
+
+local exp = redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms)
+
+if redis.call('ZSCORE', KEYS[1], ARGV[1]) then
+ return {1, exp, 0}
+end
+
+local in_use = tonumber(redis.call('ZCARD', KEYS[1]))
+
+if in_use < tonumber(ARGV[2]) then
+ redis.call('ZADD', KEYS[1], 'NX', now_ms + tonumber(ARGV[3]), ARGV[1])
+ return {1, exp, in_use}
+else
+ local first = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')
+ return {0, exp, in_use}
+end`,
+
+ release: `
+-- KEYS[1]=leases_zset ; ARGV[1]=holder_id
+return redis.call('ZREM', KEYS[1], ARGV[1])`,
+
+ heartbeat: `
+-- KEYS[1]=leases_zset ; ARGV[1]=holder_id, ARGV[2]=lease_ttl_ms
+local t = redis.call('TIME')
+local now_ms = t[1] * 1000 + math.floor(t[2] / 1000)
+
+redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms)
+
+local curr = redis.call('ZSCORE', KEYS[1], ARGV[1])
+if not curr then
+ return 0
+end
+
+local new_expiry = now_ms + tonumber(ARGV[2])
+redis.call('ZADD', KEYS[1], 'XX', new_expiry, ARGV[1])
+
+return 1`,
+ },
+} as const;
+
+type ScriptHashes = {
+ [K in keyof typeof luaScripts]: {
+ [K2 in keyof (typeof luaScripts)[K]]: string;
+ };
+};
+
+const scripts: ScriptHashes = {} as ScriptHashes;
+
+const redis = new IORedis(process.env.REDIS_URL!, {
+ lazyConnect: true,
+ maxRetriesPerRequest: null,
+ enableReadyCheck: false,
+});
+
+let initPromise: Promise<void> | null = null;
+
+export const ensureRedis = async () => {
+ if (initPromise) return initPromise;
+ initPromise = (async () => {
+ await redis.connect();
+
+ for (const [k, v] of Object.entries(luaScripts)) {
+ scripts[k] = {};
+ for (const [k2, v2] of Object.entries(v)) {
+ const h = await redis.script("LOAD", v2);
+ scripts[k][k2] = h as string;
+ }
+ }
+ logger.info("Redis connected and scripts loaded");
+ })().catch(err => {
+ initPromise = null;
+ throw err;
+ });
+ return initPromise;
+};
+
+export const semaphoreKeys = (teamId: string) => {
+ return {
+ leases: `nuq:sema:{${teamId}}:leases`,
+ };
+};
+
+async function runScript<T>(
+ hash: string,
+ keys: string[],
+ args?: (string | number)[],
+): Promise<T> {
+ await ensureRedis();
+ return (await redis.evalsha(
+ hash,
+ keys.length,
+ ...keys,
+ ...(args || []),
+ )) as T;
+}
+
+type NuQRedis = Redis & {
+ scripts: typeof scripts;
+ runScript: typeof runScript;
+ ensure: typeof ensureRedis;
+};
+
+export const nuqRedis: NuQRedis = Object.assign(redis, {
+ scripts,
+ runScript,
+ ensure: ensureRedis,
+});
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 79c3d5c79..1e5af10e8 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -71,8 +71,7 @@ import {
withSpan,
setSpanAttributes,
} from "../../lib/otel-tracer";
-
-const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+import { ScrapeUrlResponse } from "../../scraper/scrapeURL";
configDotenv();
@@ -81,8 +80,10 @@ const jobLockExtendInterval =
const jobLockExtensionTime =
Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;
-cacheableLookup.install(http.globalAgent);
-cacheableLookup.install(https.globalAgent);
+if (require.main === module) {
+ cacheableLookup.install(http.globalAgent);
+ cacheableLookup.install(https.globalAgent);
+}
async function billScrapeJob(
job: NuQJob<any>,
@@ -169,13 +170,23 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const costTracking = new CostTracking();
+ const abortController = new AbortController();
+ const abortTimeoutHandle =
+ remainingTime !== undefined
+ ? setTimeout(
+ () =>
+ abortController.abort(
+ new ScrapeJobTimeoutError("Scrape timed out"),
+ ),
+ remainingTime,
+ )
+ : undefined;
+ const signal = abortController.signal;
+
try {
if (remainingTime !== undefined && remainingTime < 0) {
throw new ScrapeJobTimeoutError("Scrape timed out");
}
- const signal = remainingTime
- ? AbortSignal.timeout(remainingTime)
- : undefined;
if (job.data.crawl_id) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
@@ -184,20 +195,31 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
}
- const pipeline = await Promise.race([
- startWebScraperPipeline({
- job,
- costTracking,
- }),
- ...(remainingTime !== undefined
- ? [
- (async () => {
- await sleep(remainingTime);
- throw new ScrapeJobTimeoutError("Scrape timed out");
- })(),
- ]
- : []),
- ]);
+ let pipeline: ScrapeUrlResponse | null = null;
+ let timeoutHandle: NodeJS.Timeout | null = null;
+ try {
+ pipeline = await Promise.race([
+ startWebScraperPipeline({
+ job,
+ costTracking,
+ }),
+ ...(remainingTime !== undefined
+ ? [
+ (async () => {
+ await new Promise(resolve => {
+ timeoutHandle = setTimeout(resolve, remainingTime);
+ });
+
+ throw new ScrapeJobTimeoutError("Scrape timed out");
+ })(),
+ ]
+ : []),
+ ]);
+ } finally {
+ if (timeoutHandle) {
+ clearTimeout(timeoutHandle);
+ }
+ }
try {
signal?.throwIfAborted();
@@ -692,6 +714,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
job.data.internalOptions?.bypassBilling ?? false,
);
return data;
+ } finally {
+ if (abortTimeoutHandle) clearTimeout(abortTimeoutHandle);
}
}
@@ -1097,7 +1121,11 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
try {
try {
let extendLockInterval: NodeJS.Timeout | null = null;
- if (job.data?.mode !== "kickoff" && job.data?.team_id) {
+ if (
+ job.data?.mode !== "kickoff" &&
+ job.data?.team_id &&
+ !job.data.skipNuq
+ ) {
extendLockInterval = setInterval(async () => {
await pushConcurrencyLimitActiveJob(
job.data.team_id,
@@ -1144,7 +1172,7 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
}
try {
- if (process.env.GCS_BUCKET_NAME) {
+ if (process.env.GCS_BUCKET_NAME && !job.data.skipNuq) {
logger.debug("Job succeeded -- putting null in Redis");
return null;
} else {
@@ -1163,15 +1191,22 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {
}
}
} finally {
- await concurrentJobDone(job);
+ if (!job.data.skipNuq) {
+ await concurrentJobDone(job);
+ }
}
} catch (error) {
logger.warn("Job failed", { error });
Sentry.captureException(error);
- if (error instanceof TransportableError) {
- throw new Error(serializeTransportableError(error));
+
+ if (job.data.skipNuq) {
+ throw error;
} else {
- throw new Error(serializeTransportableError(new UnknownError(error)));
+ if (error instanceof TransportableError) {
+ throw new Error(serializeTransportableError(error));
+ } else {
+ throw new Error(serializeTransportableError(new UnknownError(error)));
+ }
}
}
}
diff --git a/apps/api/src/services/worker/team-semaphore.ts b/apps/api/src/services/worker/team-semaphore.ts
new file mode 100644
index 000000000..169c11470
--- /dev/null
+++ b/apps/api/src/services/worker/team-semaphore.ts
@@ -0,0 +1,202 @@
+import { isSelfHosted } from "../../lib/deployment";
+import { ScrapeJobTimeoutError, TransportableError } from "../../lib/error";
+import { logger as _logger } from "../../lib/logger";
+import { nuqRedis, semaphoreKeys } from "./redis";
+import { Gauge, Histogram, register } from "prom-client";
+
+const activeSemaphores = new Gauge({
+ name: "noq_semaphore_active",
+ help: "Number of active semaphore holders",
+});
+
+const semaphoreAcquireDuration = new Histogram({
+ name: "noq_semaphore_acquire_duration_seconds",
+ help: "Semaphore acquire time",
+ // Buckets in seconds: 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s
+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
+});
+
+const { scripts, runScript, ensure } = nuqRedis;
+
+const SEMAPHORE_TTL = 30 * 1000;
+
+async function acquire(
+ teamId: string,
+ holderId: string,
+ limit: number,
+): Promise<{ granted: boolean; count: number; removed: number }> {
+ await ensure();
+
+ const keys = semaphoreKeys(teamId);
+ const [granted, count, removed] = await runScript<[number, number, number]>(
+ scripts.semaphore.acquire,
+ [keys.leases],
+ [holderId, limit, SEMAPHORE_TTL],
+ );
+
+ return {
+ granted: granted === 1,
+ count,
+ removed,
+ };
+}
+
+async function acquireBlocking(
+ teamId: string,
+ holderId: string,
+ limit: number,
+ options: {
+ base_delay_ms: number;
+ max_delay_ms: number;
+ timeout_ms: number;
+ signal: AbortSignal;
+ },
+): Promise<{ limited: boolean; removed: number }> {
+ await ensure();
+
+ const deadline = Date.now() + options.timeout_ms;
+ const keys = semaphoreKeys(teamId);
+
+ let delay = options.base_delay_ms;
+ let totalRemoved = 0;
+ let failedOnce = false;
+
+ // Start the timer for the histogram
+ const endTimer = semaphoreAcquireDuration.startTimer();
+
+ do {
+ if (options.signal.aborted) {
+ throw new ScrapeJobTimeoutError("Scrape timed out");
+ }
+
+ if (deadline < Date.now()) {
+ throw new ScrapeJobTimeoutError("Scrape timed out");
+ }
+
+ const [granted, _count, _removed] = await runScript<
+ [number, number, number]
+ >(
+ scripts.semaphore.acquire,
+ [keys.leases],
+ [holderId, limit, SEMAPHORE_TTL],
+ );
+
+ totalRemoved++;
+
+ if (granted === 1) {
+ // Stop the timer and record the duration
+ endTimer();
+ return { limited: failedOnce, removed: totalRemoved };
+ }
+
+ failedOnce = true;
+
+ const jitter = Math.floor(
+ Math.random() * Math.max(1, Math.floor(delay / 4)),
+ );
+ await new Promise(r => setTimeout(r, delay + jitter));
+
+ delay = Math.min(options.max_delay_ms, Math.floor(delay * 1.5));
+ } while (true);
+}
+
+async function heartbeat(teamId: string, holderId: string): Promise<boolean> {
+ await ensure();
+
+ const keys = semaphoreKeys(teamId);
+ return (
+ (await runScript<number>(
+ scripts.semaphore.heartbeat,
+ [keys.leases],
+ [holderId, SEMAPHORE_TTL],
+ )) === 1
+ );
+}
+
+async function release(teamId: string, holderId: string): Promise<void> {
+ await ensure();
+
+ const keys = semaphoreKeys(teamId);
+ await runScript<number>(scripts.semaphore.release, [keys.leases], [holderId]);
+}
+
+async function count(teamId: string): Promise<number> {
+ await ensure();
+
+ const keys = semaphoreKeys(teamId);
+ const count = await nuqRedis.zcard(keys.leases);
+ return count;
+}
+
+function startHeartbeat(teamId: string, holderId: string, intervalMs: number) {
+ let stopped = false;
+
+ const promise = (async () => {
+ while (!stopped) {
+ const ok = await heartbeat(teamId, holderId);
+ if (!ok) {
+ throw new TransportableError("SCRAPE_TIMEOUT", "heartbeat_failed");
+ }
+ await new Promise(r => setTimeout(r, intervalMs));
+ }
+ return Promise.reject(
+ new Error("heartbeat loop stopped unexpectedly"),
+ ) as never;
+ })();
+
+ return {
+ promise,
+ stop() {
+ stopped = true;
+ },
+ };
+}
+
+async function withSemaphore<T>(
+ teamId: string,
+ holderId: string,
+ limit: number,
+ signal: AbortSignal,
+ timeoutMs: number,
+ func: (limited: boolean) => Promise<T>,
+): Promise<T> {
+ if (isSelfHosted() && limit <= 1) {
+ _logger.debug(`Bypassing concurrency limit for ${teamId}`, {
+ teamId,
+ jobId: holderId,
+ });
+ return await func(false);
+ }
+
+ const { limited } = await acquireBlocking(teamId, holderId, limit, {
+ base_delay_ms: 25,
+ max_delay_ms: 250,
+ timeout_ms: timeoutMs,
+ signal,
+ });
+
+ const hb = startHeartbeat(teamId, holderId, SEMAPHORE_TTL / 2);
+
+ activeSemaphores.inc();
+ try {
+ const result = await Promise.race([func(limited), hb.promise]);
+ return result;
+ } finally {
+ activeSemaphores.dec();
+ hb.stop();
+
+ await release(teamId, holderId).catch(() => {});
+ }
+}
+
+const getMetrics = async () => {
+ return register.metrics();
+};
+
+export const teamConcurrencySemaphore = {
+ acquire,
+ release,
+ withSemaphore,
+ count,
+ getMetrics,
+};
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 86f96b6b9..11dd06418 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -17,6 +17,7 @@ type ScrapeJobCommon = {
team_id: string;
zeroDataRetention: boolean;
traceContext?: SerializedTraceContext;
+ skipNuq?: boolean;
};
export type ScrapeJobData = ScrapeJobCommon &
diff --git a/apps/go-html-to-md-service/.dockerignore b/apps/go-html-to-md-service/.dockerignore
new file mode 100644
index 000000000..fde6ac73c
--- /dev/null
+++ b/apps/go-html-to-md-service/.dockerignore
@@ -0,0 +1,27 @@
+# Ignore build artifacts
+*.dylib
+*.so
+*.dll
+*.exe
+
+# Ignore test files
+*_test.go
+
+# Ignore documentation
+README.md
+*.md
+
+# Ignore git
+.git
+.gitignore
+
+# Ignore IDE files
+.vscode
+.idea
+*.swp
+*.swo
+
+# Ignore temporary files
+*.tmp
+*.log
+
diff --git a/apps/go-html-to-md-service/.gitignore b/apps/go-html-to-md-service/.gitignore
new file mode 100644
index 000000000..ea5bf9a40
--- /dev/null
+++ b/apps/go-html-to-md-service/.gitignore
@@ -0,0 +1,31 @@
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool
+*.out
+
+# Dependency directories
+vendor/
+
+# Go workspace file
+go.work
+
+# IDE specific files
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# Logs
+*.log
+
+# Binary
+html-to-markdown-service
+
diff --git a/apps/go-html-to-md-service/Dockerfile b/apps/go-html-to-md-service/Dockerfile
new file mode 100644
index 000000000..ab3909c5b
--- /dev/null
+++ b/apps/go-html-to-md-service/Dockerfile
@@ -0,0 +1,42 @@
+# Build stage
+FROM golang:1.23-alpine AS builder
+
+# Install build dependencies
+RUN apk add --no-cache git
+
+# Set working directory
+WORKDIR /app
+
+# Copy go mod files
+COPY go.mod go.sum ./
+
+# Download dependencies
+RUN go mod download
+
+# Copy source code
+COPY . .
+
+# Build the application
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o html-to-markdown-service .
+
+# Final stage
+FROM alpine:latest
+
+# Install ca-certificates for HTTPS requests
+RUN apk --no-cache add ca-certificates
+
+WORKDIR /root/
+
+# Copy the binary from builder
+COPY --from=builder /app/html-to-markdown-service .
+
+# Expose port
+EXPOSE 8080
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
+ CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1
+
+# Run the application
+CMD ["./html-to-markdown-service"]
+
diff --git a/apps/go-html-to-md-service/Makefile b/apps/go-html-to-md-service/Makefile
new file mode 100644
index 000000000..e234e23da
--- /dev/null
+++ b/apps/go-html-to-md-service/Makefile
@@ -0,0 +1,87 @@
+.PHONY: help build test run clean docker-build docker-run docker-stop install lint fmt
+
+# Default target
+help:
+ @echo "HTML to Markdown Service - Available targets:"
+ @echo " make build - Build the service binary"
+ @echo " make test - Run tests"
+ @echo " make test-cover - Run tests with coverage"
+ @echo " make run - Run the service locally"
+ @echo " make clean - Clean build artifacts"
+ @echo " make install - Install dependencies"
+ @echo " make lint - Run linter"
+ @echo " make fmt - Format code"
+ @echo " make docker-build - Build Docker image"
+ @echo " make docker-run - Run with Docker Compose"
+ @echo " make docker-stop - Stop Docker containers"
+ @echo " make docker-logs - View Docker logs"
+
+# Build the service
+build:
+ @echo "Building service..."
+ go build -o html-to-markdown-service .
+
+# Run tests
+test:
+ @echo "Running tests..."
+ go test -v ./...
+
+# Run tests with coverage
+test-cover:
+ @echo "Running tests with coverage..."
+ go test -v -cover -coverprofile=coverage.out ./...
+ go tool cover -html=coverage.out -o coverage.html
+ @echo "Coverage report generated: coverage.html"
+
+# Run the service
+run:
+ @echo "Starting service..."
+ go run .
+
+# Clean build artifacts
+clean:
+ @echo "Cleaning..."
+ rm -f html-to-markdown-service
+ rm -f coverage.out coverage.html
+ go clean
+
+# Install dependencies
+install:
+ @echo "Installing dependencies..."
+ go mod download
+ go mod tidy
+
+# Run linter (requires golangci-lint)
+lint:
+ @echo "Running linter..."
+ golangci-lint run
+
+# Format code
+fmt:
+ @echo "Formatting code..."
+ go fmt ./...
+
+# Build Docker image
+docker-build:
+ @echo "Building Docker image..."
+ docker build -t html-to-markdown-service:latest .
+
+# Run with Docker Compose
+docker-run:
+ @echo "Starting with Docker Compose..."
+ docker-compose up -d
+
+# Stop Docker containers
+docker-stop:
+ @echo "Stopping Docker containers..."
+ docker-compose down
+
+# View Docker logs
+docker-logs:
+ @echo "Viewing Docker logs..."
+ docker-compose logs -f
+
+# Run all checks before commit
+pre-commit: fmt lint test
+ @echo "Pre-commit checks passed!"
+
diff --git a/apps/go-html-to-md-service/converter.go b/apps/go-html-to-md-service/converter.go
new file mode 100644
index 000000000..0d09bc072
--- /dev/null
+++ b/apps/go-html-to-md-service/converter.go
@@ -0,0 +1,145 @@
+package main
+
+import (
+ "strings"
+ "unicode/utf8"
+
+ "github.com/PuerkitoBio/goquery"
+ md "github.com/firecrawl/html-to-markdown"
+ "github.com/firecrawl/html-to-markdown/plugin"
+ "golang.org/x/net/html"
+)
+
+// Converter handles HTML to Markdown conversion
+type Converter struct {
+ converter *md.Converter
+}
+
+// NewConverter creates a new Converter instance with pre-configured rules
+func NewConverter() *Converter {
+ converter := md.NewConverter("", true, nil)
+ converter.Use(plugin.GitHubFlavored())
+ addGenericPreRule(converter)
+
+ return &Converter{
+ converter: converter,
+ }
+}
+
+// ConvertHTMLToMarkdown converts HTML string to Markdown
+func (c *Converter) ConvertHTMLToMarkdown(html string) (string, error) {
+ return c.converter.ConvertString(html)
+}
+
+// addGenericPreRule adds a robust PRE handler that extracts nested code text
+// (e.g., tables/rows/gutters) and outputs fenced blocks with detected language.
+func addGenericPreRule(conv *md.Converter) {
+ isGutter := func(class string) bool {
+ c := strings.ToLower(class)
+ return strings.Contains(c, "gutter") || strings.Contains(c, "line-numbers")
+ }
+
+ detectLang := func(sel *goquery.Selection) string {
+ classes := sel.AttrOr("class", "")
+ lower := strings.ToLower(classes)
+ for _, part := range strings.Fields(lower) {
+ if strings.HasPrefix(part, "language-") {
+ return strings.TrimPrefix(part, "language-")
+ }
+ if strings.HasPrefix(part, "lang-") {
+ return strings.TrimPrefix(part, "lang-")
+ }
+ }
+ return ""
+ }
+
+ // Collect text recursively; insert newlines after block elements and br
+ var collect func(n *html.Node, b *strings.Builder)
+ collect = func(n *html.Node, b *strings.Builder) {
+ if n == nil {
+ return
+ }
+ switch n.Type {
+ case html.TextNode:
+ b.WriteString(n.Data)
+ case html.ElementNode:
+ name := strings.ToLower(n.Data)
+ // Skip gutters
+ if name != "" {
+ // check class attr for gutters
+ for _, a := range n.Attr {
+ if a.Key == "class" && isGutter(a.Val) {
+ return
+ }
+ }
+ }
+
+ if name == "br" {
+ b.WriteString("\n")
+ }
+
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ collect(c, b)
+ }
+
+ // Newline after block-ish wrappers to preserve lines
+ switch name {
+ case "p", "div", "li", "tr", "table", "thead", "tbody", "tfoot", "section", "article", "blockquote", "pre", "h1", "h2", "h3", "h4", "h5", "h6":
+ b.WriteString("\n")
+ }
+ }
+ }
+
+ conv.AddRules(md.Rule{
+ Filter: []string{"pre"},
+ Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {
+ // find inner <code> if present for language
+ codeSel := selec.Find("code").First()
+ lang := detectLang(codeSel)
+ if lang == "" {
+ lang = detectLang(selec)
+ }
+
+ var b strings.Builder
+ for _, n := range selec.Nodes {
+ collect(n, &b)
+ }
+ content := strings.TrimRight(b.String(), "\n")
+
+ fenceChar, _ := utf8.DecodeRuneInString(opt.Fence)
+ fence := md.CalculateCodeFence(fenceChar, content)
+ text := "\n\n" + fence + lang + "\n" + content + "\n" + fence + "\n\n"
+ return md.String(text)
+ },
+ })
+
+ // Inline code: robustly extract text and fence with backticks
+ conv.AddRules(md.Rule{
+ Filter: []string{"code"},
+ Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {
+ // If inside pre, let the PRE rule handle it
+ if selec.ParentsFiltered("pre").Length() > 0 {
+ return nil
+ }
+ var b strings.Builder
+ for _, n := range selec.Nodes {
+ collect(n, &b)
+ }
+ code := b.String()
+ // collapse multiple newlines for inline code
+ code = md.TrimTrailingSpaces(strings.ReplaceAll(code, "\r\n", "\n"))
+
+ // Choose fence length safely
+ fence := "`"
+ if strings.Contains(code, "`") {
+ fence = "``"
+ if strings.Contains(code, "``") {
+ fence = "```"
+ }
+ }
+ out := fence + code + fence
+ return md.String(out)
+ },
+ })
+}
+
diff --git a/apps/go-html-to-md-service/docker-compose.yml b/apps/go-html-to-md-service/docker-compose.yml
new file mode 100644
index 000000000..36ca2caec
--- /dev/null
+++ b/apps/go-html-to-md-service/docker-compose.yml
@@ -0,0 +1,20 @@
+version: '3.8'
+
+services:
+ html-to-markdown:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ container_name: html-to-markdown-service
+ ports:
+ - "8080:8080"
+ environment:
+ - PORT=8080
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/health"]
+ interval: 30s
+ timeout: 3s
+ retries: 3
+ start_period: 5s
+
diff --git a/apps/go-html-to-md-service/go.mod b/apps/go-html-to-md-service/go.mod
new file mode 100644
index 000000000..1fdf8e997
--- /dev/null
+++ b/apps/go-html-to-md-service/go.mod
@@ -0,0 +1,23 @@
+module github.com/firecrawl/go-html-to-md-service
+
+go 1.23.0
+
+require (
+ github.com/PuerkitoBio/goquery v1.10.3
+ github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3
+ github.com/gorilla/mux v1.8.1
+ github.com/rs/zerolog v1.33.0
+ golang.org/x/net v0.41.0
+)
+
+require (
+ github.com/andybalholm/cascadia v1.3.3 // indirect
+ github.com/kr/pretty v0.3.0 // indirect
+ github.com/mattn/go-colorable v0.1.13 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
+ golang.org/x/sys v0.33.0 // indirect
+ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+ gopkg.in/yaml.v2 v2.4.0 // indirect
+)
+
+replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20250917145228-b6d0a75dfdba
diff --git a/apps/go-html-to-md-service/go.sum b/apps/go-html-to-md-service/go.sum
new file mode 100644
index 000000000..e10a7c0c1
--- /dev/null
+++ b/apps/go-html-to-md-service/go.sum
@@ -0,0 +1,117 @@
+github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
+github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3 h1:lzHIpN3DszdL8V2JRK03WleWIeW2ssVmiMAbg67ES/A=
+github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3/go.mod h1:jngam+MdNp7FZkhSTlFsuA5hXY21X0+vuiGlpgo2n5o=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=
+github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U=
+github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
diff --git a/apps/go-html-to-md-service/handler.go b/apps/go-html-to-md-service/handler.go
new file mode 100644
index 000000000..f995bc54b
--- /dev/null
+++ b/apps/go-html-to-md-service/handler.go
@@ -0,0 +1,164 @@
+package main
+
+import (
+ "encoding/json"
+ "io"
+ "net/http"
+ "time"
+
+ "github.com/gorilla/mux"
+ "github.com/rs/zerolog/log"
+)
+
+const (
+ maxRequestSize = 10 * 1024 * 1024 // 10MB max request size
+)
+
+// Handler manages HTTP request handling
+type Handler struct {
+ converter *Converter
+}
+
+// NewHandler creates a new Handler instance
+func NewHandler(converter *Converter) *Handler {
+ return &Handler{
+ converter: converter,
+ }
+}
+
+// RegisterRoutes registers all HTTP routes
+func (h *Handler) RegisterRoutes(router *mux.Router) {
+ router.HandleFunc("/health", h.HealthCheck).Methods("GET")
+ router.HandleFunc("/convert", h.ConvertHTML).Methods("POST")
+ router.HandleFunc("/", h.Index).Methods("GET")
+}
+
+// HealthCheckResponse represents the health check response
+type HealthCheckResponse struct {
+ Status string `json:"status"`
+ Timestamp time.Time `json:"timestamp"`
+ Service string `json:"service"`
+}
+
+// HealthCheck handles health check requests
+func (h *Handler) HealthCheck(w http.ResponseWriter, r *http.Request) {
+ response := HealthCheckResponse{
+ Status: "healthy",
+ Timestamp: time.Now(),
+ Service: "html-to-markdown",
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ json.NewEncoder(w).Encode(response)
+}
+
+// IndexResponse represents the index page response
+type IndexResponse struct {
+ Service string `json:"service"`
+ Version string `json:"version"`
+ Description string `json:"description"`
+ Endpoints []string `json:"endpoints"`
+}
+
+// Index handles root path requests
+func (h *Handler) Index(w http.ResponseWriter, r *http.Request) {
+ response := IndexResponse{
+ Service: "HTML to Markdown Converter",
+ Version: "1.0.0",
+ Description: "A service for converting HTML content to Markdown format",
+ Endpoints: []string{
+ "GET /health - Health check endpoint",
+ "POST /convert - Convert HTML to Markdown",
+ },
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ json.NewEncoder(w).Encode(response)
+}
+
+// ConvertRequest represents the conversion request payload
+type ConvertRequest struct {
+ HTML string `json:"html"`
+}
+
+// ConvertResponse represents the conversion response payload
+type ConvertResponse struct {
+ Markdown string `json:"markdown"`
+ Success bool `json:"success"`
+}
+
+// ErrorResponse represents an error response
+type ErrorResponse struct {
+ Error string `json:"error"`
+ Success bool `json:"success"`
+}
+
+// ConvertHTML handles HTML to Markdown conversion requests
+func (h *Handler) ConvertHTML(w http.ResponseWriter, r *http.Request) {
+ startTime := time.Now()
+
+ // Limit request body size
+ r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
+
+ // Read and decode request body
+ body, err := io.ReadAll(r.Body)
+ if err != nil {
+ log.Error().Err(err).Msg("Failed to read request body")
+ h.sendError(w, "Failed to read request body", http.StatusBadRequest)
+ return
+ }
+
+ var req ConvertRequest
+ if err := json.Unmarshal(body, &req); err != nil {
+ log.Error().Err(err).Msg("Failed to parse request body")
+ h.sendError(w, "Invalid JSON in request body", http.StatusBadRequest)
+ return
+ }
+
+ // Validate input
+ if req.HTML == "" {
+ h.sendError(w, "HTML field is required", http.StatusBadRequest)
+ return
+ }
+
+ // Convert HTML to Markdown
+ markdown, err := h.converter.ConvertHTMLToMarkdown(req.HTML)
+ if err != nil {
+ log.Error().Err(err).Msg("Failed to convert HTML to Markdown")
+ h.sendError(w, "Failed to convert HTML to Markdown", http.StatusInternalServerError)
+ return
+ }
+
+ // Log metrics
+ duration := time.Since(startTime)
+ log.Info().
+ Dur("duration_ms", duration).
+ Int("input_size", len(req.HTML)).
+ Int("output_size", len(markdown)).
+ Msg("HTML to Markdown conversion completed")
+
+ // Send response
+ response := ConvertResponse{
+ Markdown: markdown,
+ Success: true,
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusOK)
+ json.NewEncoder(w).Encode(response)
+}
+
+// sendError sends an error response
+func (h *Handler) sendError(w http.ResponseWriter, message string, statusCode int) {
+ response := ErrorResponse{
+ Error: message,
+ Success: false,
+ }
+
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(statusCode)
+ json.NewEncoder(w).Encode(response)
+}
+
diff --git a/apps/go-html-to-md-service/handler_test.go b/apps/go-html-to-md-service/handler_test.go
new file mode 100644
index 000000000..7ea308682
--- /dev/null
+++ b/apps/go-html-to-md-service/handler_test.go
@@ -0,0 +1,261 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "github.com/gorilla/mux"
+)
+
+func TestHealthCheck(t *testing.T) {
+ converter := NewConverter()
+ handler := NewHandler(converter)
+
+ router := mux.NewRouter()
+ handler.RegisterRoutes(router)
+
+ req, err := http.NewRequest("GET", "/health", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rr := httptest.NewRecorder()
+ router.ServeHTTP(rr, req)
+
+ if status := rr.Code; status != http.StatusOK {
+ t.Errorf("handler returned wrong status code: got %v want %v",
+ status, http.StatusOK)
+ }
+
+ var response HealthCheckResponse
+ if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {
+ t.Errorf("failed to parse response: %v", err)
+ }
+
+ if response.Status != "healthy" {
+ t.Errorf("handler returned unexpected status: got %v want %v",
+ response.Status, "healthy")
+ }
+
+ if response.Service != "html-to-markdown" {
+ t.Errorf("handler returned unexpected service: got %v want %v",
+ response.Service, "html-to-markdown")
+ }
+}
+
+func TestIndex(t *testing.T) {
+ converter := NewConverter()
+ handler := NewHandler(converter)
+
+ router := mux.NewRouter()
+ handler.RegisterRoutes(router)
+
+ req, err := http.NewRequest("GET", "/", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ rr := httptest.NewRecorder()
+ router.ServeHTTP(rr, req)
+
+ if status := rr.Code; status != http.StatusOK {
+ t.Errorf("handler returned wrong status code: got %v want %v",
+ status, http.StatusOK)
+ }
+
+ var response IndexResponse
+ if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {
+ t.Errorf("failed to parse response: %v", err)
+ }
+
+ if response.Service != "HTML to Markdown Converter" {
+ t.Errorf("handler returned unexpected service name")
+ }
+
+ if len(response.Endpoints) == 0 {
+ t.Errorf("handler returned no endpoints")
+ }
+}
+
+func TestConvertHTML_Success(t *testing.T) {
+ converter := NewConverter()
+ handler := NewHandler(converter)
+
+ router := mux.NewRouter()
+ handler.RegisterRoutes(router)
+
+ testCases := []struct {
+ name string
+ html string
+ expectedOutput string
+ }{
+ {
+ name: "Simple paragraph",
+ html: "<p>Hello, World!</p>",
+ expectedOutput: "Hello, World!",
+ },
+ {
+ name: "Bold text",
+ html: "<p>This is <strong>bold</strong> text</p>",
+ expectedOutput: "**bold**",
+ },
+ {
+ name: "Link",
+ html: "<a href='https://example.com'>Example</a>",
+ expectedOutput: "[Example](https://example.com)",
+ },
+ {
+ name: "Code block",
+ html: "<pre><code>console.log('hello');</code></pre>",
+ },
+ {
+ name: "Inline code",
+ html: "<code>const x = 1;</code>",
+ expectedOutput: "`const x = 1;`",
+ },
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ reqBody := ConvertRequest{
+ HTML: tc.html,
+ }
+ jsonBody, _ := json.Marshal(reqBody)
+
+ req, err := http.NewRequest("POST", "/convert", bytes.NewBuffer(jsonBody))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ rr := httptest.NewRecorder()
+ router.ServeHTTP(rr, req)
+
+ if status := rr.Code; status != http.StatusOK {
+ t.Errorf("handler returned wrong status code: got %v want %v",
+ status, http.StatusOK)
+ }
+
+ var response ConvertResponse
+ if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {
+ t.Errorf("failed to parse response: %v", err)
+ }
+
+ if !response.Success {
+ t.Errorf("conversion was not successful")
+ }
+
+ if tc.expectedOutput != "" && response.Markdown == "" {
+ t.Errorf("expected markdown output, got empty string")
+ }
+
+ // For simple checks, verify expected output is contained in response
+ if tc.expectedOutput != "" {
+ if !contains(response.Markdown, tc.expectedOutput) {
+ t.Errorf("expected markdown to contain %q, got %q",
+ tc.expectedOutput, response.Markdown)
+ }
+ }
+ })
+ }
+}
+
+func TestConvertHTML_EmptyHTML(t *testing.T) {
+ converter := NewConverter()
+ handler := NewHandler(converter)
+
+ router := mux.NewRouter()
+ handler.RegisterRoutes(router)
+
+ reqBody := ConvertRequest{
+ HTML: "",
+ }
+ jsonBody, _ := json.Marshal(reqBody)
+
+ req, err := http.NewRequest("POST", "/convert", bytes.NewBuffer(jsonBody))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ rr := httptest.NewRecorder()
+ router.ServeHTTP(rr, req)
+
+ if status := rr.Code; status != http.StatusBadRequest {
+ t.Errorf("handler returned wrong status code: got %v want %v",
+ status, http.StatusBadRequest)
+ }
+
+ var response ErrorResponse
+ if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {
+ t.Errorf("failed to parse response: %v", err)
+ }
+
+ if response.Success {
+ t.Errorf("expected success to be false")
+ }
+}
+
+func TestConvertHTML_InvalidJSON(t *testing.T) {
+ converter := NewConverter()
+ handler := NewHandler(converter)
+
+ router := mux.NewRouter()
+ handler.RegisterRoutes(router)
+
+ req, err := http.NewRequest("POST", "/convert", bytes.NewBuffer([]byte("invalid json")))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Content-Type", "application/json")
+
+ rr := httptest.NewRecorder()
+ router.ServeHTTP(rr, req)
+
+ if status := rr.Code; status != http.StatusBadRequest {
+ t.Errorf("handler returned wrong status code: got %v want %v",
+ status, http.StatusBadRequest)
+ }
+}
+
+func TestConverter_ComplexHTML(t *testing.T) {
+ converter := NewConverter()
+
+ testHTML := `
+ <div>
+ <h1>Title</h1>
+ <p>This is a paragraph with <strong>bold</strong> and <em>italic</em> text.</p>
+ <ul>
+ <li>Item 1</li>
+ <li>Item 2</li>
+ </ul>
+ <pre><code class="language-javascript">console.log('hello');</code></pre>
+ </div>
+ `
+
+ markdown, err := converter.ConvertHTMLToMarkdown(testHTML)
+ if err != nil {
+ t.Errorf("conversion failed: %v", err)
+ }
+
+ if markdown == "" {
+ t.Errorf("expected non-empty markdown output")
+ }
+
+ // Verify key elements are present
+ expectedElements := []string{"Title", "bold", "italic", "Item 1", "console.log"}
+ for _, elem := range expectedElements {
+ if !contains(markdown, elem) {
+ t.Errorf("expected markdown to contain %q, but it didn't", elem)
+ }
+ }
+}
+
+// Helper function to check if a string contains a substring
+func contains(s, substr string) bool {
+ return bytes.Contains([]byte(s), []byte(substr))
+}
+
diff --git a/apps/go-html-to-md-service/main.go b/apps/go-html-to-md-service/main.go
new file mode 100644
index 000000000..1da6e217d
--- /dev/null
+++ b/apps/go-html-to-md-service/main.go
@@ -0,0 +1,100 @@
+package main
+
+import (
+ "context"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+
+ "github.com/gorilla/mux"
+ "github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
+)
+
+const (
+ defaultPort = "8080"
+ defaultShutdownTimeout = 30 * time.Second
+ defaultReadTimeout = 30 * time.Second
+ defaultWriteTimeout = 30 * time.Second
+ maxUploadSize = 50 * 1024 * 1024
+)
+
+func main() {
+ // Configure logging
+ zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
+
+ env := os.Getenv("ENV")
+
+ if env == "production" {
+ zerolog.SetGlobalLevel(zerolog.InfoLevel)
+ } else {
+ log.Logger = log.Output(zerolog.ConsoleWriter{
+ Out: os.Stdout,
+ TimeFormat: time.RFC3339,
+ })
+ }
+
+ // Get port from environment or use default
+ port := os.Getenv("PORT")
+ if port == "" {
+ port = defaultPort
+ }
+
+ // Initialize converter
+ converter := NewConverter()
+
+ // Initialize handlers
+ handler := NewHandler(converter)
+
+ // Setup router
+ router := mux.NewRouter()
+
+ router.Use(func(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize)
+ next.ServeHTTP(w, r)
+ })
+ })
+
+ handler.RegisterRoutes(router)
+
+ // Create server
+ srv := &http.Server{
+ Addr: ":" + port,
+ Handler: router,
+ ReadTimeout: defaultReadTimeout,
+ WriteTimeout: defaultWriteTimeout,
+ }
+
+ // Start server in goroutine
+ go func() {
+ log.Info().
+ Str("port", port).
+ Str("env", env).
+ Msg("Starting HTML to Markdown service")
+
+ if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ log.Fatal().Err(err).Msg("Failed to start server")
+ }
+ }()
+
+ // Wait for interrupt signal to gracefully shutdown the server
+ quit := make(chan os.Signal, 1)
+ signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)
+ <-quit
+
+ log.Info().Msg("Shutting down server...")
+
+ // Create shutdown context with timeout
+ ctx, cancel := context.WithTimeout(context.Background(), defaultShutdownTimeout)
+ defer cancel()
+
+ // Attempt graceful shutdown
+ if err := srv.Shutdown(ctx); err != nil {
+ log.Fatal().Err(err).Msg("Server forced to shutdown")
+ }
+
+ log.Info().Msg("Server exited")
+}
diff --git a/apps/go-html-to-md-service/requests.http b/apps/go-html-to-md-service/requests.http
new file mode 100644
index 000000000..6ac797d9a
--- /dev/null
+++ b/apps/go-html-to-md-service/requests.http
@@ -0,0 +1,224 @@
+### HTML to Markdown Service - API Tests
+### Use the REST Client extension in VS Code or IntelliJ to run these requests
+
+@baseUrl = http://localhost:8080
+
+### Health Check
+GET {{baseUrl}}/health
+
+###
+
+### Service Info
+GET {{baseUrl}}/
+
+###
+
+### Test 1: Simple HTML Conversion
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<h1>Hello World</h1>"
+}
+
+###
+
+### Test 2: Paragraph with Bold and Italic
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<p>This is a <strong>bold</strong> and <em>italic</em> test.</p>"
+}
+
+###
+
+### Test 3: Unordered List
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<ul><li>First item</li><li>Second item</li><li>Third item</li></ul>"
+}
+
+###
+
+### Test 4: Ordered List
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<ol><li>Step 1</li><li>Step 2</li><li>Step 3</li></ol>"
+}
+
+###
+
+### Test 5: Code Block with Language
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<pre><code class=\"language-javascript\">function hello() {\n console.log('Hello, World!');\n}</code></pre>"
+}
+
+###
+
+### Test 6: Inline Code
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<p>Use the <code>console.log()</code> function to debug.</p>"
+}
+
+###
+
+### Test 7: Links and Anchors
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<p>Visit <a href=\"https://example.com\">our website</a> for more information.</p>"
+}
+
+###
+
+### Test 8: Images
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<img src=\"https://example.com/image.jpg\" alt=\"Example Image\">"
+}
+
+###
+
+### Test 9: Table
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<table><thead><tr><th>Name</th><th>Age</th><th>City</th></tr></thead><tbody><tr><td>John</td><td>30</td><td>New York</td></tr><tr><td>Jane</td><td>25</td><td>London</td></tr></tbody></table>"
+}
+
+###
+
+### Test 10: Blockquote
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<blockquote><p>This is a blockquote.</p></blockquote>"
+}
+
+###
+
+### Test 11: Horizontal Rule
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<p>Before</p><hr><p>After</p>"
+}
+
+###
+
+### Test 12: Nested Structure
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<div><h1>Title</h1><p>Paragraph with <strong>bold</strong> text.</p><ul><li>Item 1</li><li>Item 2</li></ul></div>"
+}
+
+###
+
+### Test 13: Complex Document
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<article><header><h1>Article Title</h1><p class=\"meta\">By Author Name</p></header><section><h2>Introduction</h2><p>This is the introduction with <strong>bold</strong> and <em>italic</em> text.</p><pre><code class=\"language-python\">def hello():\n print('Hello, World!')</code></pre></section><section><h2>Conclusion</h2><p>Visit <a href=\"https://example.com\">example.com</a> for more.</p></section></article>"
+}
+
+###
+
+### Test 14: Strikethrough (GitHub Flavored Markdown)
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<p>This is <del>deleted</del> text.</p>"
+}
+
+###
+
+### Test 15: Task List (GitHub Flavored Markdown)
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<ul><li><input type=\"checkbox\" checked> Completed task</li><li><input type=\"checkbox\"> Incomplete task</li></ul>"
+}
+
+###
+
+### Test 16: Code Block with Multiple Languages
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<div><pre><code class=\"language-javascript\">console.log('JS');</code></pre><pre><code class=\"language-python\">print('Python')</code></pre><pre><code class=\"language-go\">fmt.Println(\"Go\")</code></pre></div>"
+}
+
+###
+
+### Test 17: Mixed Content
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<div><h2>Tutorial</h2><p>First, install the package:</p><pre><code class=\"language-bash\">npm install package-name</code></pre><p>Then use it in your code:</p><pre><code class=\"language-javascript\">const pkg = require('package-name');</code></pre><p>For more info, see <a href=\"https://docs.example.com\">the docs</a>.</p></div>"
+}
+
+###
+
+### Error Test 1: Empty HTML
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": ""
+}
+
+###
+
+### Error Test 2: Invalid JSON
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{invalid json}
+
+###
+
+### Error Test 3: Missing HTML Field
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "notHtml": "<p>Test</p>"
+}
+
+###
+
+### Performance Test: Large HTML (1000 paragraphs)
+# Warning: This may take a few seconds
+POST {{baseUrl}}/convert
+Content-Type: application/json
+
+{
+ "html": "<div><h1>Large Document</h1><p>Paragraph 1</p><p>Paragraph 2</p><p>Paragraph 3</p><p>Paragraph 4</p><p>Paragraph 5</p><p>Paragraph 6</p><p>Paragraph 7</p><p>Paragraph 8</p><p>Paragraph 9</p><p>Paragraph 10</p><h2>Section</h2><p>More content here with <strong>bold</strong> and <em>italic</em> text.</p><ul><li>Item 1</li><li>Item 2</li><li>Item 3</li></ul><pre><code class=\"language-javascript\">console.log('code');</code></pre></div>"
+}
+
+###
+