Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: noq scrape system (#2419)
delong38a8d64996356
diff --git a/.github/workflows/deploy-go-service.yaml b/.github/workflows/deploy-go-service.yamlnew file mode 100644index 000000000..3c2b8c729--- /dev/null+++ b/.github/workflows/deploy-go-service.yaml@@ -0,0 +1,42 @@+name: Deploy Go Service to GHCR++env:+ DOTNET_VERSION: "6.0.x"++on:+ push:+ branches:+ - main+ paths:+ - apps/go-html-to-md-service/**+ - .github/workflows/deploy-go-service.yaml+ workflow_dispatch:++jobs:+ push-app-image:+ runs-on: ubuntu-latest+ defaults:+ run:+ working-directory: "./apps/go-html-to-md-service"+ steps:+ - name: "Checkout GitHub Action"+ uses: actions/checkout@main++ - name: "Set up Docker Buildx"+ uses: docker/setup-buildx-action@v3++ - name: "Login to GitHub Container Registry"+ uses: docker/login-action@v3+ with:+ registry: ghcr.io+ username: ${{github.actor}}+ password: ${{secrets.GITHUB_TOKEN}}++ - name: "Build and Push Image"+ uses: docker/build-push-action@v6+ with:+ context: ./apps/go-html-to-md-service+ push: true+ tags: ghcr.io/firecrawl/go-html-to-md-service:latest+ cache-from: type=registry,ref=ghcr.io/firecrawl/go-html-to-md-service:buildcache+ cache-to: type=registry,ref=ghcr.io/firecrawl/go-html-to-md-service:buildcache,mode=maxdiff --git a/.gitignore b/.gitignoreindex 9a97f69a3..1c2469ae5 100644--- a/.gitignore+++ b/.gitignore@@ -49,3 +49,6 @@ CLAUDE.local.mdapps/python-sdk/.venv//apps/api/running-docs/+++/apps/go-html-to-md-service/.gomodcache/\ No newline at end of filediff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex 37d18f12b..d2e122d7d 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -14,7 +14,7 @@ kuchikiki = "0.8.2"lol_html = "2.6.0"lopdf = "0.38.0"maud = "0.27.0"-napi = { version = "3.0.0", features = ["serde-json"] }+napi = { version = "3.0.0", features = ["serde-json", "tokio_rt"] }napi-derive = "3.0.0"nodesig = { git = "https://github.com/firecrawl/nodesig" }psl = "2.1.140"@@ -26,6 +26,7 @@ texting_robots = "0.2.2"url = "2.5.7"zip = "5.0.0"calamine = "0.26"+tokio = "1.48.0"[build-dependencies]napi-build = "2"diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex fa8244cf1..d301481ad 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -7,6 +7,7 @@ use std::{sync::LazyLock,};use texting_robots::Robot;+use tokio::task;use url::Url;static FILE_EXTENSIONS: &[&str] = &[@@ -173,13 +174,7 @@ fn no_sections(url_str: &str) -> bool {#[inline]fn is_non_web_protocol(url_str: &str) -> bool {const NON_WEB_PROTOCOLS: &[&str] = &[- "mailto:",- "tel:",- "telnet:",- "ftp:",- "ftps:",- "ssh:",- "file:",+ "mailto:", "tel:", "telnet:", "ftp:", "ftps:", "ssh:", "file:",];NON_WEB_PROTOCOLS@@ -360,10 +355,12 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult&& is_subdomain(&url, &base_url){// When allowing subdomains, still honor include patterns- let match_target = if data.regex_on_full_url { url_str } else { path };- if !includes_regex.is_empty()- && !includes_regex.iter().any(|r| r.is_match(match_target))- {+ let match_target = if data.regex_on_full_url {+ url_str+ } else {+ path+ };+ if !includes_regex.is_empty() && !includes_regex.iter().any(|r| r.is_match(match_target)) {denial_reasons.insert(link, INCLUDE_PATTERN.to_string());continue;}@@ -383,9 +380,17 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult/// Filter links based on crawling rules and constraints.#[napi]-pub fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {- _filter_links(data)- .map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))+pub async fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {+ let res = task::spawn_blocking(move || _filter_links(data))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("filter_links join error: {e}"),+ )+ })?;++ res.map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))}fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, String> {@@ -553,9 +558,17 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri/// Filter a single URL based on crawling rules and constraints.#[napi]-pub fn filter_url(data: FilterUrlCall) -> Result<FilterUrlResult> {- _filter_url(data)- .map_err(|e| Error::new(Status::GenericFailure, format!("Filter URL error: {e}")))+pub async fn filter_url(data: FilterUrlCall) -> Result<FilterUrlResult> {+ let res = task::spawn_blocking(move || _filter_url(data))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("filter_url join error: {e}"),+ )+ })?;++ res.map_err(|e| Error::new(Status::GenericFailure, format!("Filter URL error: {e}")))}fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, String> {@@ -616,8 +629,17 @@ fn _parse_sitemap_xml(xml_content: &str) -> std::result::Result<ParsedSitemap, S/// Parse XML sitemap content into structured data.#[napi]-pub fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {- _parse_sitemap_xml(&xml_content).map_err(|e| {+pub async fn parse_sitemap_xml(xml_content: String) -> Result<ParsedSitemap> {+ let res = task::spawn_blocking(move || _parse_sitemap_xml(&xml_content))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("parse_sitemap_xml join error: {e}"),+ )+ })?;++ res.map_err(|e| {Error::new(Status::GenericFailure,format!("Parse sitemap XML error: {e}"),@@ -700,13 +722,17 @@ fn _process_sitemap(xml_content: &str) -> std::result::Result<SitemapProcessingR/// Process sitemap XML and extract crawling instructions.#[napi]-pub fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResult> {- _process_sitemap(&xml_content).map_err(|e| {- Error::new(- Status::GenericFailure,- format!("Process sitemap error: {e}"),- )- })+pub async fn process_sitemap(xml_content: String) -> Result<SitemapProcessingResult> {+ let res = task::spawn_blocking(move || _process_sitemap(&xml_content))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("process_sitemap join error: {e}"),+ )+ })?;++ res.map_err(|e| Error::new(Status::GenericFailure, format!("Parse sitemap error: {e}")))}#[cfg(test)]@@ -1006,8 +1032,7 @@ mod tests {// And should exclude blog due to includePathsassert!(result.denial_reasons- .get("https://sub.example.com/blog")- .is_some());+ .contains_key("https://sub.example.com/blog"));assert_eq!(result.denial_reasonsdiff --git a/apps/api/native/src/html.rs b/apps/api/native/src/html.rsindex 6d548c3a0..e1afab37f 100644--- a/apps/api/native/src/html.rs+++ b/apps/api/native/src/html.rs@@ -5,6 +5,7 @@ use napi_derive::napi;use nodesig::{get_node_signature, SignatureMode};use serde::{Deserialize, Serialize};use serde_json::Value;+use tokio::task;use url::Url;use crate::utils::to_napi_err;@@ -12,7 +13,7 @@ use crate::utils::to_napi_err;fn _extract_base_href_from_document(document: &NodeRef,url: &Url,-) -> Result<String, Box<dyn std::error::Error>> {+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {if let Some(base) = document.select("base[href]").map_err(|_| "Failed to select base href".to_string())?@@ -27,7 +28,10 @@ fn _extract_base_href_from_document(Ok(url.to_string())}-fn _extract_base_href(html: &str, url: &str) -> Result<String, Box<dyn std::error::Error>> {+fn _extract_base_href(+ html: &str,+ url: &str,+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {let document = parse_html().one(html);let url = Url::parse(url)?;_extract_base_href_from_document(&document, &url)@@ -35,43 +39,61 @@ fn _extract_base_href(html: &str, url: &str) -> Result<String, Box<dyn std::erro/// Extract the base href from HTML document.#[napi]-pub fn extract_base_href(html: String, url: String) -> napi::Result<String> {- _extract_base_href(&html, &url).map_err(to_napi_err)+pub async fn extract_base_href(html: String, url: String) -> napi::Result<String> {+ let res = task::spawn_blocking(move || _extract_base_href(&html, &url))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("extract_base_href join error: {e}"),+ )+ })?;++ res.map_err(to_napi_err)}/// Extract all links from HTML document.#[napi]-pub fn extract_links(html: Option<String>) -> napi::Result<Vec<String>> {- let html = match html {- Some(h) => h,- None => return Ok(Vec::new()),- };+pub async fn extract_links(html: Option<String>) -> napi::Result<Vec<String>> {+ task::spawn_blocking(move || {+ let html = match html {+ Some(h) => h,+ None => return Ok(Vec::new()),+ };- let document = parse_html().one(html.as_str());+ let document = parse_html().one(html.as_str());- let anchors: Vec<_> = document- .select("a[href]")- .map_err(|_| to_napi_err("Failed to select links"))?- .collect();+ let anchors: Vec<_> = document+ .select("a[href]")+ .map_err(|_| to_napi_err("Failed to select links"))?+ .collect();- let mut out: Vec<String> = Vec::new();+ let mut out: Vec<String> = Vec::new();- for anchor in anchors {- let mut href = match anchor.attributes.borrow().get("href") {- Some(x) => x.to_string(),- None => continue,- };+ for anchor in anchors {+ let mut href = match anchor.attributes.borrow().get("href") {+ Some(x) => x.to_string(),+ None => continue,+ };- if href.starts_with("http:/") && !href.starts_with("http://") {- href = format!("http://{}", &href[6..]);- } else if href.starts_with("https:/") && !href.starts_with("https://") {- href = format!("https://{}", &href[7..]);- }+ if href.starts_with("http:/") && !href.starts_with("http://") {+ href = format!("http://{}", &href[6..]);+ } else if href.starts_with("https:/") && !href.starts_with("https://") {+ href = format!("https://{}", &href[7..]);+ }- out.push(href);- }+ out.push(href);+ }- Ok(out)+ Ok(out)+ })+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("extract_links join error: {e}"),+ )+ })?}macro_rules! insert_meta_name {@@ -112,7 +134,9 @@ macro_rules! insert_meta_property {};}-fn _extract_metadata(html: &str) -> Result<HashMap<String, Value>, Box<dyn std::error::Error>> {+fn _extract_metadata(+ html: &str,+) -> Result<HashMap<String, Value>, Box<dyn std::error::Error + Send + Sync>> {let document = parse_html().one(html);let mut out = HashMap::<String, Value>::new();@@ -267,13 +291,22 @@ fn _extract_metadata(html: &str) -> Result<HashMap<String, Value>, Box<dyn std::/// Extract metadata from HTML document.#[napi]-pub fn extract_metadata(html: Option<String>) -> napi::Result<HashMap<String, Value>> {- let html = match html {- Some(h) => h,- None => return Ok(HashMap::new()),- };+pub async fn extract_metadata(html: Option<String>) -> napi::Result<HashMap<String, Value>> {+ task::spawn_blocking(move || {+ let html = match html {+ Some(h) => h,+ None => return Ok(HashMap::new()),+ };- _extract_metadata(&html).map_err(to_napi_err)+ _extract_metadata(&html).map_err(to_napi_err)+ })+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("extract_metadata join error: {e}"),+ )+ })?}const EXCLUDE_NON_MAIN_TAGS: [&str; 42] = [@@ -356,7 +389,9 @@ struct ImageSource {is_x: bool,}-fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn std::error::Error>> {+fn _transform_html_inner(+ opts: TransformHtmlOptions,+) -> Result<String, Box<dyn std::error::Error + Send + Sync>> {let mut document = parse_html().one(opts.html.as_ref());let url = Url::parse(&_extract_base_href_from_document(&document,@@ -574,8 +609,17 @@ fn _transform_html_inner(opts: TransformHtmlOptions) -> Result<String, Box<dyn s/// Transform and clean HTML content based on provided options.#[napi]-pub fn transform_html(opts: TransformHtmlOptions) -> napi::Result<String> {- _transform_html_inner(opts).map_err(to_napi_err)+pub async fn transform_html(opts: TransformHtmlOptions) -> napi::Result<String> {+ let res = task::spawn_blocking(move || _transform_html_inner(opts))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("transform_html join error: {e}"),+ )+ })?;++ res.map_err(to_napi_err)}fn _get_inner_json(html: &str) -> Result<String, ()> {@@ -584,8 +628,17 @@ fn _get_inner_json(html: &str) -> Result<String, ()> {/// Extract inner text content from HTML body.#[napi]-pub fn get_inner_json(html: String) -> napi::Result<String> {- _get_inner_json(&html).map_err(|_| to_napi_err("Failed to get inner JSON"))+pub async fn get_inner_json(html: String) -> napi::Result<String> {+ let res = task::spawn_blocking(move || _get_inner_json(&html))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("get_inner_json join error: {e}"),+ )+ })?;++ res.map_err(|_| to_napi_err("Failed to get inner JSON"))}#[derive(Deserialize, Serialize)]@@ -612,7 +665,7 @@ pub struct ExtractedAttributeResult {fn _extract_attributes(html: &str,options: &ExtractAttributesOptions,-) -> Result<Vec<ExtractedAttributeResult>, Box<dyn std::error::Error>> {+) -> Result<Vec<ExtractedAttributeResult>, Box<dyn std::error::Error + Send + Sync>> {let document = parse_html().one(html);let mut results = Vec::new();@@ -659,21 +712,33 @@ fn _extract_attributes(/// Extract specified attributes from HTML elements matching selectors.#[napi]-pub fn extract_attributes(+pub async fn extract_attributes(html: String,options: ExtractAttributesOptions,) -> napi::Result<Vec<ExtractedAttributeResult>> {- _extract_attributes(&html, &options).map_err(to_napi_err)+ let res = task::spawn_blocking(move || _extract_attributes(&html, &options))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("extract_attributes join error: {e}"),+ )+ })?;++ res.map_err(to_napi_err)}-fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn std::error::Error>> {+fn _extract_images(+ html: &str,+ base_url: &str,+) -> Result<Vec<String>, Box<dyn std::error::Error + Send + Sync>> {let document = parse_html().one(html);let base_url = Url::parse(base_url)?;let base_href = _extract_base_href_from_document(&document, &base_url)?;let base_href_url = Url::parse(&base_href)?;let mut images = HashSet::<String>::new();- let resolve_image_url = |src: &str| -> Result<String, Box<dyn std::error::Error>> {+ let resolve_image_url = |src: &str| -> Result<String, Box<dyn std::error::Error + Send + Sync>> {if src.starts_with("data:") || src.starts_with("blob:") {return Ok(src.to_string());}@@ -812,6 +877,98 @@ fn _extract_images(html: &str, base_url: &str) -> Result<Vec<String>, Box<dyn st/// Extract all image URLs from HTML document.#[napi]-pub fn extract_images(html: String, base_url: String) -> napi::Result<Vec<String>> {- _extract_images(&html, &base_url).map_err(to_napi_err)+pub async fn extract_images(html: String, base_url: String) -> napi::Result<Vec<String>> {+ let res = task::spawn_blocking(move || _extract_images(&html, &base_url))+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("extract_images join error: {e}"),+ )+ })?;++ res.map_err(to_napi_err)+}++/// Process multi-line links in markdown.+#[napi]+pub async fn post_process_markdown(markdown: String) -> napi::Result<String> {+ let res = task::spawn_blocking(move || {+ let mut link_open_count = 0usize;+ let mut out = String::with_capacity(markdown.len());++ for ch in markdown.chars() {+ match ch {+ '[' => {+ link_open_count += 1;+ }+ ']' => {+ link_open_count = link_open_count.saturating_sub(1);+ }+ _ => {}+ }++ let inside_link_content = link_open_count > 0;+ if inside_link_content && ch == '\n' {+ out.push('\\');+ out.push('\n');+ } else {+ out.push(ch);+ }+ }++ remove_skip_to_content_links(&out)+ })+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("post_process_markdown join error: {e}"),+ )+ })?;++ Ok(res)+}++fn remove_skip_to_content_links(input: &str) -> String {+ const LABEL: &str = "Skip to Content";+ let bytes = input.as_bytes();+ let len = bytes.len();+ let mut out = String::with_capacity(len);+ let mut i = 0;++ 'outer: while i < len {+ if bytes[i] == b'[' {+ let label_start = i + 1;+ let label_end = label_start + LABEL.len();++ if label_end <= len && bytes[label_start..label_end].iter().all(|b| b.is_ascii()) {+ let label_slice = &input[label_start..label_end];++ if label_slice.eq_ignore_ascii_case(LABEL)+ && label_end + 3 <= len+ && bytes[label_end] == b']'+ && bytes[label_end + 1] == b'('+ && bytes[label_end + 2] == b'#'+ {+ let mut j = label_end + 3;++ while j < len {+ let ch = input[j..].chars().next().unwrap();+ if ch == ')' {+ i = j + ch.len_utf8();+ continue 'outer;+ }+ j += ch.len_utf8();+ }+ }+ }+ }++ let ch = input[i..].chars().next().unwrap();+ out.push(ch);+ i += ch.len_utf8();+ }++ out}diff --git a/apps/api/package.json b/apps/api/package.jsonindex fd68c0474..862388993 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -126,6 +126,7 @@"pg": "^8.16.3","posthog-node": "^4.0.1","prettier": "^3.6.2",+ "prom-client": "^15.1.3","psl": "^1.15.0","rate-limiter-flexible": "2.4.2","redlock": "5.0.0-beta.2",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex c714383ce..abd8388da 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -191,6 +191,9 @@ importers:prettier:specifier: ^3.6.2version: 3.6.2+ prom-client:+ specifier: ^15.1.3+ version: 15.1.3psl:specifier: ^1.15.0version: 1.15.0@@ -3731,6 +3734,9 @@ packages:resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==}+ resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}+resolution: {integrity: sha512-v2YAxEmKaBLahNwE1mjp4WON6huMNeuDvagFZW+ASCuA/ku0bXR9hSMw0XpiqMoA3+rmnyck/tPRSFQkoC9Cuw==}@@ -5922,6 +5928,10 @@ packages:resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==}engines: {node: '>=0.4.0'}+ resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==}+ engines: {node: ^16 || ^18 || >=20}+resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==}engines: {node: '>= 6'}@@ -6465,6 +6475,9 @@ packages:os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android]hasBin: true+ resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}+resolution: {integrity: sha512-3ZnLvgWF29jikg1sAQ1g0o+lr5JX6sVgYvfUJazn7ZjJroDBUTWp44/+cFVX0bULjv4vci+rBD+oGVAkWqhUbw==}engines: {node: '>=18'}@@ -11919,6 +11932,8 @@ snapshots:dependencies:file-uri-to-path: 1.0.0+ [email protected]: {}+@@ -14480,6 +14495,11 @@ snapshots:+ dependencies:+ '@opentelemetry/api': 1.9.0+ tdigest: 0.1.2+dependencies:kleur: 3.0.3@@ -15067,6 +15087,10 @@ snapshots:+ dependencies:+ bintrees: 1.0.2+dependencies:http-proxy-agent: 5.0.0diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex fdbfd086c..23137225b 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -1,7 +1,11 @@import { parseApi } from "../lib/parseApi";import { getRateLimiter } from "../services/rate-limiter";import { AuthResponse, NotificationType, RateLimiterMode } from "../types";-import { supabase_rr_service, supabase_service } from "../services/supabase";+import {+ supabase_acuc_only_service,+ supabase_rr_service,+ supabase_service,+} from "../services/supabase";import { withAuth } from "../lib/withAuth";import { RateLimiterRedis } from "rate-limiter-flexible";import { sendNotification } from "../services/notification/email_notification";@@ -12,20 +16,7 @@ import { setValue } from "../services/redis";import { validate } from "uuid";import * as Sentry from "@sentry/node";import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types";-// const { data, error } = await supabase_service-// .from('api_keys')-// .select(`-// key,-// team_id,-// teams (-// subscriptions (-// price_id-// )-// )-// `)-// .eq('key', normalizedApi)-// .limit(1)-// .single();+function normalizedApiIsUuid(potentialUuid: string): boolean {// Check if the string is a valid UUIDreturn validate(potentialUuid);@@ -166,7 +157,10 @@ export async function getACUC(return acuc;}- if (process.env.USE_DB_AUTHENTICATION !== "true") {+ if (+ process.env.USE_DB_AUTHENTICATION !== "true" &&+ !process.env.SUPABASE_ACUC_URL+ ) {const acuc = mockACUC();acuc.is_extract = isExtract;return acuc;@@ -187,8 +181,11 @@ export async function getACUC(let retries = 0;const maxRetries = 5;while (retries < maxRetries) {- const client =- Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;+ const client = !!process.env.SUPABASE_ACUC_URL+ ? supabase_acuc_only_service+ : Math.random() > 2 / 3+ ? supabase_rr_service+ : supabase_service;({ data, error } = await client.rpc("auth_credit_usage_chunk_36",{@@ -292,7 +289,10 @@ export async function getACUCTeam(return acuc;}- if (process.env.USE_DB_AUTHENTICATION !== "true") {+ if (+ process.env.USE_DB_AUTHENTICATION !== "true" &&+ !process.env.SUPABASE_ACUC_URL+ ) {const acuc = mockACUC();acuc.is_extract = isExtract;return acuc;@@ -314,8 +314,11 @@ export async function getACUCTeam(const maxRetries = 5;while (retries < maxRetries) {- const client =- Math.random() > 2 / 3 ? supabase_rr_service : supabase_service;+ const client = !!process.env.SUPABASE_ACUC_URL+ ? supabase_acuc_only_service+ : Math.random() > 2 / 3+ ? supabase_rr_service+ : supabase_service;({ data, error } = await client.rpc("auth_credit_usage_chunk_36_from_team",{@@ -393,6 +396,10 @@ export async function authenticateUser(res,mode?: RateLimiterMode,): Promise<AuthResponse> {+ if (!!process.env.SUPABASE_ACUC_URL) {+ return supaAuthenticateUser(req, res, mode);+ }+return withAuth(supaAuthenticateUser, {success: true,chunk: null,diff --git a/apps/api/src/controllers/v0/admin/metrics.ts b/apps/api/src/controllers/v0/admin/metrics.tsindex 4174d2372..fbaeb6ae3 100644--- a/apps/api/src/controllers/v0/admin/metrics.ts+++ b/apps/api/src/controllers/v0/admin/metrics.ts@@ -1,6 +1,7 @@import type { Request, Response } from "express";import { getRedisConnection } from "../../../services/queue-service";import { nuqGetLocalMetrics, scrapeQueue } from "../../../services/worker/nuq";+import { teamConcurrencySemaphore } from "../../../services/worker/team-semaphore";export async function metricsController(_: Request, res: Response) {let cursor: string = "0";@@ -26,6 +27,8 @@ export async function metricsController(_: Request, res: Response) {}} while (cursor !== "0");+ const semaphoreMetrics = await teamConcurrencySemaphore.getMetrics();+res.contentType("text/plain").send(`\# HELP concurrency_limit_queue_job_count The number of jobs in the concurrency limit queue per team# TYPE concurrency_limit_queue_job_count gauge@@ -35,9 +38,8 @@ ${Object.entries(metrics)`concurrency_limit_queue_job_count{team_id="${key}"} ${value}`,).join("\n")}-${nuqGetLocalMetrics()}-`);+${semaphoreMetrics}`);}export async function nuqMetricsController(_: Request, res: Response) {diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex ea3ba4ccc..3d3908884 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -383,6 +383,8 @@ export async function mapController(});let result: Awaited<ReturnType<typeof getMapResults>>;+ let timeoutHandle: NodeJS.Timeout | null = null;+const abort = new AbortController();try {result = (await Promise.race([@@ -405,11 +407,12 @@ export async function mapController(}),...(req.body.timeout !== undefined? [- new Promise((resolve, reject) =>- setTimeout(() => {- abort.abort(new MapTimeoutError());- reject(new MapTimeoutError());- }, req.body.timeout),+ new Promise(+ (_resolve, reject) =>+ (timeoutHandle = setTimeout(() => {+ abort.abort(new MapTimeoutError());+ reject(new MapTimeoutError());+ }, req.body.timeout)),),]: []),@@ -424,6 +427,10 @@ export async function mapController(} else {throw error;}+ } finally {+ if (timeoutHandle) {+ clearTimeout(timeoutHandle);+ }}// Bill the teamdiff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 84523d2da..aca00a019 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -8,12 +8,15 @@ import {ScrapeResponse,} from "./types";import { v4 as uuidv4 } from "uuid";-import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";import { fromV1ScrapeOptions } from "../v2/types";import { TransportableError } from "../../lib/error";-import { scrapeQueue } from "../../services/worker/nuq";+import { NuQJob } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";+import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";+import { processJobInternal } from "../../services/worker/scrape-worker";+import { ScrapeJobData } from "../../types";+import { AbortManagerThrownError } from "../../scraper/scrapeURL/lib/abortManager";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -42,6 +45,7 @@ export async function scrapeController(const logger = _logger.child({method: "scrapeController",jobId,+ noq: true,scrapeId: jobId,teamId: req.auth.team_id,team_id: req.auth.team_id,@@ -61,7 +65,7 @@ export async function scrapeController(const origin = req.body.origin;const timeout = req.body.timeout;- const startTime = new Date().getTime();+ // const startTime = new Date().getTime();const isDirectToBullMQ =process.env.SEARCH_PREVIEW_TOKEN !== undefined &&@@ -73,43 +77,6 @@ export async function scrapeController(req.auth.team_id,);- const jobPriority = await getJobPriority({- team_id: req.auth.team_id,- basePriority: 10,- });-- const bullJob = await addScrapeJob(- {- url: req.body.url,- mode: "single_urls",- team_id: req.auth.team_id,- scrapeOptions,- internalOptions: {- ...internalOptions,- teamId: req.auth.team_id,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME- ? true- : false,- unnormalizedSourceURL: preNormalizedBody.url,- bypassBilling: isDirectToBullMQ,- zeroDataRetention,- teamFlags: req.acuc?.flags ?? null,- },- origin,- integration: req.body.integration,- startTime: controllerStartTime,- zeroDataRetention: zeroDataRetention ?? false,- apiKeyId: req.acuc?.api_key_id ?? null,- },- jobId,- jobPriority,- isDirectToBullMQ,- true,- );- logger.info(- "Added scrape job now" + (bullJob ? "" : " (to concurrency queue)"),- );-const totalWait =(req.body.waitFor ?? 0) +(req.body.actions ?? []).reduce(@@ -117,22 +84,80 @@ export async function scrapeController(0,);- let doc: Document;+ let timeoutHandle: NodeJS.Timeout | null = null;+ let doc: Document | null = null;try {- doc = await waitForJob(- bullJob ? bullJob : jobId,- timeout + totalWait,- zeroDataRetention ?? false,- logger,+ const lockStart = Date.now();+ const aborter = new AbortController();+ if (timeout) {+ timeoutHandle = setTimeout(() => {+ aborter.abort();+ }, timeout * 0.667);+ }+ req.on("close", () => aborter.abort());++ doc = await teamConcurrencySemaphore.withSemaphore(+ req.auth.team_id,+ jobId,+ req.acuc?.concurrency || 1,+ aborter.signal,+ timeout ?? 60_000,+ async () => {+ const jobPriority = await getJobPriority({+ team_id: req.auth.team_id,+ basePriority: 10,+ });++ const lockTime = Date.now() - lockStart;++ logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {+ teamId: req.auth.team_id,+ lockTime,+ });++ const job: NuQJob<ScrapeJobData> = {+ id: jobId,+ status: "active",+ createdAt: new Date(),+ priority: jobPriority,+ data: {+ url: req.body.url,+ mode: "single_urls",+ team_id: req.auth.team_id,+ scrapeOptions,+ internalOptions: {+ ...internalOptions,+ teamId: req.auth.team_id,+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME+ ? true+ : false,+ unnormalizedSourceURL: preNormalizedBody.url,+ bypassBilling: isDirectToBullMQ,+ zeroDataRetention,+ teamFlags: req.acuc?.flags ?? null,+ },+ skipNuq: true,+ origin,+ integration: req.body.integration,+ startTime: controllerStartTime,+ zeroDataRetention: zeroDataRetention ?? false,+ apiKeyId: req.acuc?.api_key_id ?? null,+ },+ };++ const doc = await processJobInternal(job);+ return doc ?? null;+ },);} catch (e) {- logger.error(`Error in scrapeController`, {- version: "v1",- error: e,- });+ const timeoutErr =+ e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";- if (zeroDataRetention) {- await scrapeQueue.removeJob(jobId, logger);+ if (!timeoutErr) {+ logger.error(`Error in scrapeController`, {+ version: "v1",+ error: e,+ });}if (e instanceof TransportableError) {@@ -157,12 +182,14 @@ export async function scrapeController(error: `(Internal server error) - ${e && e.message ? e.message : e}`,});}+ } finally {+ if (timeoutHandle) {+ clearTimeout(timeoutHandle);+ }}logger.info("Done with waitForJob");- await scrapeQueue.removeJob(jobId, logger);-logger.info("Removed job from queue");if (!req.body.formats.includes("rawHtml")) {@@ -186,7 +213,7 @@ export async function scrapeController(return res.status(200).json({success: true,- data: doc,+ data: doc!,scrape_id: origin?.includes("website") ? jobId : undefined,});}diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex c3b5e5505..ee7d4692a 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -174,7 +174,15 @@ export async function crawlStatusController(res: Response<CrawlStatusResponse>,isBatch = false,) {- const isPreviewTeam = req.auth.team_id?.startsWith("preview");+ const uuidReg =+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+ if (!req.params.jobId || !uuidReg.test(req.params.jobId)) {+ return res.status(400).json({+ success: false,+ error: "Invalid job ID",+ });+ }+const start =typeof req.query.skip === "string" ? parseInt(req.query.skip, 10) : 0;const end =diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex bcc788147..652401cd3 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -54,6 +54,8 @@ export async function mapController(});let result: MapResult;+ let timeoutHandle: NodeJS.Timeout | null = null;+const abort = new AbortController();try {result = (await Promise.race([@@ -78,11 +80,12 @@ export async function mapController(}),...(req.body.timeout !== undefined? [- new Promise((resolve, reject) =>- setTimeout(() => {- abort.abort(new MapTimeoutError());- reject(new MapTimeoutError());- }, req.body.timeout),+ new Promise(+ (_resolve, reject) =>+ (timeoutHandle = setTimeout(() => {+ abort.abort(new MapTimeoutError());+ reject(new MapTimeoutError());+ }, req.body.timeout)),),]: []),@@ -97,6 +100,10 @@ export async function mapController(} else {throw error;}+ } finally {+ if (timeoutHandle) {+ clearTimeout(timeoutHandle);+ }}// Bill the team@@ -142,6 +149,8 @@ export async function mapController(num_tokens: 0,credits_billed: 1,zeroDataRetention: false, // not supported+ }).catch(error => {+ logger.error(`Failed to log job for team ${req.auth.team_id}: ${error}`);});// Log final timing informationdiff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.tsindex a1064b4a7..5e0252025 100644--- a/apps/api/src/controllers/v2/scrape-status.ts+++ b/apps/api/src/controllers/v2/scrape-status.ts@@ -3,6 +3,15 @@ import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";export async function scrapeStatusController(req: any, res: any) {+ const uuidReg =+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;+ if (!req.params.jobId || !uuidReg.test(req.params.jobId)) {+ return res.status(400).json({+ success: false,+ error: "Invalid crawl ID",+ });+ }+const logger = _logger.child({module: "scrape-status",method: "scrapeStatusController",diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 7d3bd08c7..bbc80c1ca 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -8,13 +8,15 @@ import {ScrapeResponse,} from "./types";import { v4 as uuidv4 } from "uuid";-import { addScrapeJob, waitForJob } from "../../services/queue-jobs";-import { getJobPriority } from "../../lib/job-priority";import { hasFormatOfType } from "../../lib/format-utils";import { TransportableError } from "../../lib/error";-import { scrapeQueue } from "../../services/worker/nuq";+import { NuQJob } from "../../services/worker/nuq";import { checkPermissions } from "../../lib/permissions";import { withSpan, setSpanAttributes, SpanKind } from "../../lib/otel-tracer";+import { processJobInternal } from "../../services/worker/scrape-worker";+import { ScrapeJobData } from "../../types";+import { teamConcurrencySemaphore } from "../../services/worker/team-semaphore";+import { getJobPriority } from "../../lib/job-priority";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -78,6 +80,7 @@ export async function scrapeController(const logger = _logger.child({method: "scrapeController",jobId,+ noq: true,scrapeId: jobId,teamId: req.auth.team_id,team_id: req.auth.team_id,@@ -107,59 +110,6 @@ export async function scrapeController(process.env.SEARCH_PREVIEW_TOKEN !== undefined &&process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;- const jobPriority = await getJobPriority({- team_id: req.auth.team_id,- basePriority: 10,- });-- // Job enqueuing span- const job = await withSpan(- "api.scrape.enqueue_job",- async enqueueSpan => {- const result = await addScrapeJob(- {- url: req.body.url,- mode: "single_urls",- team_id: req.auth.team_id,- scrapeOptions: {- ...req.body,- ...(req.body.__experimental_cache- ? {- maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours- }- : {}),- },- internalOptions: {- teamId: req.auth.team_id,- saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME- ? true- : false,- unnormalizedSourceURL: preNormalizedBody.url,- bypassBilling: isDirectToBullMQ,- zeroDataRetention,- teamFlags: req.acuc?.flags ?? null,- },- origin,- integration: req.body.integration,- startTime: controllerStartTime,- zeroDataRetention,- apiKeyId: req.acuc?.api_key_id ?? null,- },- jobId,- jobPriority,- isDirectToBullMQ,- true,- );-- setSpanAttributes(enqueueSpan, {- "job.priority": jobPriority,- "job.direct_to_bullmq": isDirectToBullMQ,- });-- return result;- },- );-const totalWait =(req.body.waitFor ?? 0) +(req.body.actions ?? []).reduce(@@ -167,33 +117,110 @@ export async function scrapeController(0,);- let doc: Document;+ let doc: Document | null = null;+ let timeoutHandle: NodeJS.Timeout | null = null;try {- // Wait for job completion span- doc = await withSpan("api.scrape.wait_for_job", async waitSpan => {- setSpanAttributes(waitSpan, {- "wait.timeout": timeout !== undefined ? timeout + totalWait : null,- "wait.job_id": jobId,- });+ const lockStart = Date.now();+ const aborter = new AbortController();+ if (timeout) {+ // Semaphore has 2/3 of the timeout time to get a lock to allow for scrape time+ timeoutHandle = setTimeout(() => {+ aborter.abort();+ }, timeout * 0.667);+ }+ req.on("close", () => aborter.abort());++ doc = await teamConcurrencySemaphore.withSemaphore(+ req.auth.team_id,+ jobId,+ req.acuc?.concurrency || 1,+ aborter.signal,+ timeout ?? 60_000,+ async () => {+ const jobPriority = await getJobPriority({+ team_id: req.auth.team_id,+ basePriority: 10,+ });- const result = await waitForJob(- job ?? jobId,- timeout !== undefined ? timeout + totalWait : null,- zeroDataRetention,- logger,- );+ // TODO: send 429 on abort+ const lockTime = Date.now() - lockStart;- setSpanAttributes(waitSpan, {- "wait.success": true,- });+ logger.debug(`Lock acquired for team: ${req.auth.team_id}`, {+ teamId: req.auth.team_id,+ lockTime,+ });- return result;- });+ // Wait for job completion span+ const doc = await withSpan(+ "api.scrape.wait_for_job",+ async waitSpan => {+ setSpanAttributes(waitSpan, {+ "wait.timeout":+ timeout !== undefined ? timeout + totalWait : null,+ "wait.job_id": jobId,+ });++ const job: NuQJob<ScrapeJobData> = {+ id: jobId,++ status: "active",+ createdAt: new Date(),+ priority: jobPriority,+ data: {+ url: req.body.url,+ mode: "single_urls",+ team_id: req.auth.team_id,+ scrapeOptions: {+ ...req.body,+ ...(req.body.__experimental_cache+ ? {+ maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours+ }+ : {}),+ },+ internalOptions: {+ teamId: req.auth.team_id,+ saveScrapeResultToGCS: process.env+ .GCS_FIRE_ENGINE_BUCKET_NAME+ ? true+ : false,+ unnormalizedSourceURL: preNormalizedBody.url,+ bypassBilling: isDirectToBullMQ,+ zeroDataRetention,+ teamFlags: req.acuc?.flags ?? null,+ },+ skipNuq: true,+ origin,+ integration: req.body.integration,+ startTime: controllerStartTime,+ zeroDataRetention,+ apiKeyId: req.acuc?.api_key_id ?? null,+ },+ };++ const result = await processJobInternal(job);++ setSpanAttributes(waitSpan, {+ "wait.success": true,+ });++ return result ?? null;+ },+ );++ return doc;+ },+ );} catch (e) {- logger.error(`Error in scrapeController`, {- version: "v2",- error: e,- });+ const timeoutErr =+ e instanceof TransportableError && e.code === "SCRAPE_TIMEOUT";++ if (!timeoutErr) {+ logger.error(`Error in scrapeController`, {+ version: "v2",+ error: e,+ });+ }setSpanAttributes(span, {"scrape.error": e instanceof Error ? e.message : String(e),@@ -201,10 +228,6 @@ export async function scrapeController(e instanceof TransportableError ? e.code : "unknown",});- if (zeroDataRetention) {- await scrapeQueue.removeJob(jobId, logger);- }-if (e instanceof TransportableError) {// DNS resolution errors should return 200 with success: falseif (e.code === "SCRAPE_DNS_RESOLUTION_ERROR") {@@ -236,10 +259,12 @@ export async function scrapeController(error: `(Internal server error) - ${e && e.message ? e.message : e}`,});}+ } finally {+ if (timeoutHandle) {+ clearTimeout(timeoutHandle);+ }}- await scrapeQueue.removeJob(jobId, logger);-if (!hasFormatOfType(req.body.formats, "rawHtml")) {if (doc && doc.rawHtml) {delete doc.rawHtml;@@ -273,7 +298,7 @@ export async function scrapeController(return res.status(200).json({success: true,- data: doc,+ data: doc!,scrape_id: origin?.includes("website") ? jobId : undefined,});},diff --git a/apps/api/src/lib/extract/extraction-service.ts b/apps/api/src/lib/extract/extraction-service.tsindex d3570d362..d59a68968 100644--- a/apps/api/src/lib/extract/extraction-service.ts+++ b/apps/api/src/lib/extract/extraction-service.ts@@ -492,9 +492,9 @@ export async function performExtraction(ajv.compile(multiEntitySchema);// Wrap in timeout promise- const timeoutPromise = new Promise(resolve => {- setTimeout(() => resolve(null), timeoutCompletion);- });+ // const timeoutPromise = new Promise(resolve => {+ // setTimeout(() => resolve(null), timeoutCompletion);+ // });const completionPromise = batchExtractPromise({diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex d79e64ce6..a4850d19a 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -439,8 +439,9 @@ export async function performExtraction_F0(ajv.compile(multiEntitySchema);// Wrap in timeout promise+ let timeoutHandle: NodeJS.Timeout;const timeoutPromise = new Promise(resolve => {- setTimeout(() => resolve(null), timeoutCompletion);+ timeoutHandle = setTimeout(() => resolve(null), timeoutCompletion);});// Check if page should be extracted before proceeding@@ -504,7 +505,9 @@ export async function performExtraction_F0(const multiEntityCompletion = (await Promise.race([completionPromise,timeoutPromise,- ])) as Awaited<ReturnType<typeof generateCompletions_F0>>;+ ]).finally(() => {+ clearTimeout(timeoutHandle);+ })) as Awaited<ReturnType<typeof generateCompletions_F0>>;// Track multi-entity extraction tokensif (multiEntityCompletion) {diff --git a/apps/api/src/lib/extract/fire-0/reranker-f0.ts b/apps/api/src/lib/extract/fire-0/reranker-f0.tsindex 58e33e8fd..c295d7f77 100644--- a/apps/api/src/lib/extract/fire-0/reranker-f0.ts+++ b/apps/api/src/lib/extract/fire-0/reranker-f0.ts@@ -78,8 +78,9 @@ export async function rerankLinksWithLLM_F0(for (let retry = 0; retry <= MAX_RETRIES; retry++) {try {+ let timeoutHandle: NodeJS.Timeout;const timeoutPromise = new Promise<null>(resolve => {- setTimeout(() => resolve(null), TIMEOUT_MS);+ timeoutHandle = setTimeout(() => resolve(null), TIMEOUT_MS);});// dumpToFile(new Date().toISOString(),[buildRerankerSystemPrompt(), buildRerankerUserPrompt(searchQuery), schema, linksContent])@@ -114,7 +115,9 @@ export async function rerankLinksWithLLM_F0(const completion = await Promise.race([completionPromise,timeoutPromise,- ]);+ ]).finally(() => {+ clearTimeout(timeoutHandle);+ });if (!completion) {// console.log(`Chunk ${chunkIndex + 1}: Timeout on attempt ${retry + 1}`);diff --git a/apps/api/src/lib/extract/reranker.ts b/apps/api/src/lib/extract/reranker.tsindex 44a1b8b30..092a0d059 100644--- a/apps/api/src/lib/extract/reranker.ts+++ b/apps/api/src/lib/extract/reranker.ts@@ -100,9 +100,9 @@ export async function rerankLinksWithLLM(for (let retry = 0; retry <= MAX_RETRIES; retry++) {try {- const timeoutPromise = new Promise<null>(resolve => {- setTimeout(() => resolve(null), TIMEOUT_MS);- });+ // const timeoutPromise = new Promise<null>(resolve => {+ // setTimeout(() => resolve(null), TIMEOUT_MS);+ // });const systemPrompt = `You are analyzing URLs for ${isMultiEntity ? "collecting multiple items" : "specific information"}.The user's query is: ${searchQuery}diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.tsnew file mode 100644index 000000000..0541f4234--- /dev/null+++ b/apps/api/src/lib/html-to-markdown-client.ts@@ -0,0 +1,113 @@+/**+ * HTTP Client for HTML to Markdown conversion service+ *+ * This client communicates with the Go-based HTML to Markdown microservice+ * to avoid blocking Node.js event loop with heavy conversions.+ */++import axios, { AxiosInstance, AxiosError } from "axios";+import { logger } from "./logger";+import * as Sentry from "@sentry/node";++interface ConvertRequest {+ html: string;+}++interface ConvertResponse {+ markdown: string;+ success: boolean;+}++interface ErrorResponse {+ error: string;+ success: boolean;+}++/**+ * Convert HTML to Markdown using direct axios call+ * @param html HTML string to convert+ * @param serviceUrl URL of the HTML to Markdown service (default: http://localhost:8080)+ * @returns Markdown string+ * @throws Error if conversion fails+ */+export async function convertHTMLToMarkdownWithHttpService(+ html: string,+ serviceUrl?: string,+): Promise<string> {+ if (!html || html.trim() === "") {+ return "";+ }++ const url =+ serviceUrl ||+ process.env.HTML_TO_MARKDOWN_SERVICE_URL ||+ "http://localhost:8080";+ const startTime = Date.now();++ try {+ const request: ConvertRequest = { html };++ const response = await axios.post<ConvertResponse>(+ `${url}/convert`,+ request,+ {+ timeout: 30000, // 30 second timeout+ headers: {+ "Content-Type": "application/json",+ },+ },+ );++ const duration = Date.now() - startTime;++ if (!response.data.success) {+ throw new Error("Conversion was not successful");+ }++ logger.debug("HTML to Markdown conversion successful", {+ duration_ms: duration,+ input_size: html.length,+ output_size: response.data.markdown.length,+ });++ return response.data.markdown;+ } catch (error) {+ const duration = Date.now() - startTime;++ if (axios.isAxiosError(error)) {+ const axiosError = error as AxiosError<ErrorResponse>;++ const errorMessage =+ axiosError.response?.data?.error || axiosError.message;+ const statusCode = axiosError.response?.status;++ logger.error("HTML to Markdown conversion failed", {+ error: errorMessage,+ statusCode,+ duration_ms: duration,+ serviceUrl: url,+ });++ // Capture in Sentry with additional context+ Sentry.captureException(error, {+ tags: {+ service: "html-to-markdown",+ status_code: statusCode,+ },+ extra: {+ serviceUrl: url,+ errorMessage,+ inputSize: html.length,+ },+ });++ throw new Error(`HTML to Markdown conversion failed: ${errorMessage}`);+ } else {+ logger.error("Unexpected error during HTML to Markdown conversion", {+ error,+ });+ Sentry.captureException(error);+ throw error;+ }+ }+}diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.tsindex 5a21b9b1f..04b162bda 100644--- a/apps/api/src/lib/html-to-markdown.ts+++ b/apps/api/src/lib/html-to-markdown.ts@@ -6,6 +6,8 @@ import dotenv from "dotenv";import { logger } from "./logger";import { stat } from "fs/promises";import { HTML_TO_MARKDOWN_PATH } from "../natives";+import { convertHTMLToMarkdownWithHttpService } from "./html-to-markdown-client";+import { postProcessMarkdown } from "@mendable/firecrawl-rs";dotenv.config();// TODO: add a timeout to the Go parser@@ -57,13 +59,30 @@ export async function parseMarkdown(return "";}+ // Try HTTP service first if enabled+ if (process.env.HTML_TO_MARKDOWN_SERVICE_URL) {+ try {+ let markdownContent = await convertHTMLToMarkdownWithHttpService(html);+ markdownContent = await postProcessMarkdown(markdownContent);+ return markdownContent;+ } catch (error) {+ logger.error(+ "Error converting HTML to Markdown with HTTP service, falling back to original parser",+ { error },+ );+ Sentry.captureException(error, {+ tags: {+ fallback: "original_parser",+ },+ });+ }+ }+try {if (process.env.USE_GO_MARKDOWN_PARSER == "true") {const converter = await GoMarkdownConverter.getInstance();let markdownContent = await converter.convertHTMLToMarkdown(html);-- markdownContent = processMultiLineLinks(markdownContent);- markdownContent = removeSkipToContentLinks(markdownContent);+ markdownContent = await postProcessMarkdown(markdownContent);// logger.info(`HTML to Markdown conversion using Go parser successful`);return markdownContent;}@@ -108,8 +127,7 @@ export async function parseMarkdown(try {let markdownContent = await turndownService.turndown(html);- markdownContent = processMultiLineLinks(markdownContent);- markdownContent = removeSkipToContentLinks(markdownContent);+ markdownContent = await postProcessMarkdown(markdownContent);return markdownContent;} catch (error) {diff --git a/apps/api/src/lib/retry-utils.ts b/apps/api/src/lib/retry-utils.tsindex eada3ceec..6246e8a9c 100644--- a/apps/api/src/lib/retry-utils.ts+++ b/apps/api/src/lib/retry-utils.ts@@ -54,7 +54,7 @@ export async function attemptRequest<T>(* Abortable sleep function that resolves immediately if the signal is aborted*/function abortableSleep(ms: number, signal?: AbortSignal): Promise<void> {- return new Promise((resolve, reject) => {+ return new Promise(resolve => {if (signal?.aborted) {resolve();return;diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 770624b45..1ddea4ef5 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -49,68 +49,68 @@ v1Router.use(requestTimingMiddleware("v1"));// Configure payment middleware to enable micropayment-protected endpoints// This middleware handles payment verification and processing for premium API features// x402 payments protocol - https://github.com/coinbase/x402-v1Router.use(- paymentMiddleware(- (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||- "0x0000000000000000000000000000000000000000",- {- "POST /x402/search": {- price: process.env.X402_ENDPOINT_PRICE_USD as string,- network: process.env.X402_NETWORK as- | "base-sepolia"- | "base"- | "avalanche-fuji"- | "avalanche"- | "iotex",- config: {- discoverable: true,- description:- "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",- mimeType: "application/json",- maxTimeoutSeconds: 120,- inputSchema: {- body: {- query: {- type: "string",- description: "Search query to find relevant web pages",- required: true,- },- limit: {- type: "number",- description: "Maximum number of results to return (max 10)",- required: false,- },- scrapeOptions: {- type: "object",- description: "Options for scraping the found pages",- required: false,- },- },- },- outputSchema: {- type: "object",- properties: {- success: { type: "boolean" },- data: {- type: "array",- items: {- type: "object",- properties: {- url: { type: "string" },- title: { type: "string" },- description: { type: "string" },- markdown: { type: "string" },- },- },- },- },- },- },- },- },- facilitator,- ),-);+// v1Router.use(+// paymentMiddleware(+// (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||+// "0x0000000000000000000000000000000000000000",+// {+// "POST /x402/search": {+// price: process.env.X402_ENDPOINT_PRICE_USD as string,+// network: process.env.X402_NETWORK as+// | "base-sepolia"+// | "base"+// | "avalanche-fuji"+// | "avalanche"+// | "iotex",+// config: {+// discoverable: true,+// description:+// "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",+// mimeType: "application/json",+// maxTimeoutSeconds: 120,+// inputSchema: {+// body: {+// query: {+// type: "string",+// description: "Search query to find relevant web pages",+// required: true,+// },+// limit: {+// type: "number",+// description: "Maximum number of results to return (max 10)",+// required: false,+// },+// scrapeOptions: {+// type: "object",+// description: "Options for scraping the found pages",+// required: false,+// },+// },+// },+// outputSchema: {+// type: "object",+// properties: {+// success: { type: "boolean" },+// data: {+// type: "array",+// items: {+// type: "object",+// properties: {+// url: { type: "string" },+// title: { type: "string" },+// description: { type: "string" },+// markdown: { type: "string" },+// },+// },+// },+// },+// },+// },+// },+// },+// facilitator,+// ),+// );v1Router.post("/scrape",@@ -310,5 +310,65 @@ v1Router.post("/x402/search",authMiddleware(RateLimiterMode.Search),countryCheck,+ paymentMiddleware(+ (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||+ "0x0000000000000000000000000000000000000000",+ {+ "POST /x402/search": {+ price: process.env.X402_ENDPOINT_PRICE_USD as string,+ network: process.env.X402_NETWORK as+ | "base-sepolia"+ | "base"+ | "avalanche-fuji"+ | "avalanche"+ | "iotex",+ config: {+ discoverable: true,+ description:+ "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",+ mimeType: "application/json",+ maxTimeoutSeconds: 120,+ inputSchema: {+ body: {+ query: {+ type: "string",+ description: "Search query to find relevant web pages",+ required: true,+ },+ limit: {+ type: "number",+ description: "Maximum number of results to return (max 10)",+ required: false,+ },+ scrapeOptions: {+ type: "object",+ description: "Options for scraping the found pages",+ required: false,+ },+ },+ },+ outputSchema: {+ type: "object",+ properties: {+ success: { type: "boolean" },+ data: {+ type: "array",+ items: {+ type: "object",+ properties: {+ url: { type: "string" },+ title: { type: "string" },+ description: { type: "string" },+ markdown: { type: "string" },+ },+ },+ },+ },+ },+ },+ },+ },+ facilitator,+ ),wrap(x402SearchController),);diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 86b82ef63..87d58d767 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -44,117 +44,117 @@ v2Router.use(requestTimingMiddleware("v2"));// Configure payment middleware to enable micropayment-protected endpoints// This middleware handles payment verification and processing for premium API features// x402 payments protocol - https://github.com/coinbase/x402-v2Router.use(- paymentMiddleware(- (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||- "0x0000000000000000000000000000000000000000",- {- "POST /x402/search": {- price: process.env.X402_ENDPOINT_PRICE_USD as string,- network: process.env.X402_NETWORK as- | "base-sepolia"- | "base"- | "avalanche-fuji"- | "avalanche"- | "iotex",- config: {- discoverable: true,- description:- "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",- mimeType: "application/json",- maxTimeoutSeconds: 120,- inputSchema: {- body: {- query: {- type: "string",- description: "Search query to find relevant web pages",- required: true,- },- sources: {- type: "array",- description: "Sources to search (web, news, images)",- required: false,- },- limit: {- type: "number",- description: "Maximum number of results to return (max 10)",- required: false,- },- scrapeOptions: {- type: "object",- description: "Options for scraping the found pages",- required: false,- },- asyncScraping: {- type: "boolean",- description: "Whether to return job IDs for async scraping",- required: false,- },- },- },- outputSchema: {- type: "object",- properties: {- success: { type: "boolean" },- data: {- type: "object",- properties: {- web: {- type: "array",- items: {- type: "object",- properties: {- url: { type: "string" },- title: { type: "string" },- description: { type: "string" },- markdown: { type: "string" },- },- },- },- news: {- type: "array",- items: {- type: "object",- properties: {- url: { type: "string" },- title: { type: "string" },- snippet: { type: "string" },- markdown: { type: "string" },- },- },- },- images: {- type: "array",- items: {- type: "object",- properties: {- url: { type: "string" },- title: { type: "string" },- markdown: { type: "string" },- },- },- },- },- },- scrapeIds: {- type: "object",- description:- "Job IDs for async scraping (if asyncScraping is true)",- properties: {- web: { type: "array", items: { type: "string" } },- news: { type: "array", items: { type: "string" } },- images: { type: "array", items: { type: "string" } },- },- },- creditsUsed: { type: "number" },- },- },- },- },- },- facilitator,- ),-);+// v2Router.use(+// paymentMiddleware(+// (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||+// "0x0000000000000000000000000000000000000000",+// {+// "POST /x402/search": {+// price: process.env.X402_ENDPOINT_PRICE_USD as string,+// network: process.env.X402_NETWORK as+// | "base-sepolia"+// | "base"+// | "avalanche-fuji"+// | "avalanche"+// | "iotex",+// config: {+// discoverable: true,+// description:+// "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",+// mimeType: "application/json",+// maxTimeoutSeconds: 120,+// inputSchema: {+// body: {+// query: {+// type: "string",+// description: "Search query to find relevant web pages",+// required: true,+// },+// sources: {+// type: "array",+// description: "Sources to search (web, news, images)",+// required: false,+// },+// limit: {+// type: "number",+// description: "Maximum number of results to return (max 10)",+// required: false,+// },+// scrapeOptions: {+// type: "object",+// description: "Options for scraping the found pages",+// required: false,+// },+// asyncScraping: {+// type: "boolean",+// description: "Whether to return job IDs for async scraping",+// required: false,+// },+// },+// },+// outputSchema: {+// type: "object",+// properties: {+// success: { type: "boolean" },+// data: {+// type: "object",+// properties: {+// web: {+// type: "array",+// items: {+// type: "object",+// properties: {+// url: { type: "string" },+// title: { type: "string" },+// description: { type: "string" },+// markdown: { type: "string" },+// },+// },+// },+// news: {+// type: "array",+// items: {+// type: "object",+// properties: {+// url: { type: "string" },+// title: { type: "string" },+// snippet: { type: "string" },+// markdown: { type: "string" },+// },+// },+// },+// images: {+// type: "array",+// items: {+// type: "object",+// properties: {+// url: { type: "string" },+// title: { type: "string" },+// markdown: { type: "string" },+// },+// },+// },+// },+// },+// scrapeIds: {+// type: "object",+// description:+// "Job IDs for async scraping (if asyncScraping is true)",+// properties: {+// web: { type: "array", items: { type: "string" } },+// news: { type: "array", items: { type: "string" } },+// images: { type: "array", items: { type: "string" } },+// },+// },+// creditsUsed: { type: "number" },+// },+// },+// },+// },+// },+// facilitator,+// ),+// );v2Router.post("/search",@@ -314,5 +314,114 @@ v2Router.post(authMiddleware(RateLimiterMode.Search),countryCheck,blocklistMiddleware,+ paymentMiddleware(+ (process.env.X402_PAY_TO_ADDRESS as `0x${string}`) ||+ "0x0000000000000000000000000000000000000000",+ {+ "POST /x402/search": {+ price: process.env.X402_ENDPOINT_PRICE_USD as string,+ network: process.env.X402_NETWORK as+ | "base-sepolia"+ | "base"+ | "avalanche-fuji"+ | "avalanche"+ | "iotex",+ config: {+ discoverable: true,+ description:+ "The search endpoint combines web search (SERP) with Firecrawl's scraping capabilities to return full page content for any query. Requires micropayment via X402 protocol",+ mimeType: "application/json",+ maxTimeoutSeconds: 120,+ inputSchema: {+ body: {+ query: {+ type: "string",+ description: "Search query to find relevant web pages",+ required: true,+ },+ sources: {+ type: "array",+ description: "Sources to search (web, news, images)",+ required: false,+ },+ limit: {+ type: "number",+ description: "Maximum number of results to return (max 10)",+ required: false,+ },+ scrapeOptions: {+ type: "object",+ description: "Options for scraping the found pages",+ required: false,+ },+ asyncScraping: {+ type: "boolean",+ description: "Whether to return job IDs for async scraping",+ required: false,+ },+ },+ },+ outputSchema: {+ type: "object",+ properties: {+ success: { type: "boolean" },+ data: {+ type: "object",+ properties: {+ web: {+ type: "array",+ items: {+ type: "object",+ properties: {+ url: { type: "string" },+ title: { type: "string" },+ description: { type: "string" },+ markdown: { type: "string" },+ },+ },+ },+ news: {+ type: "array",+ items: {+ type: "object",+ properties: {+ url: { type: "string" },+ title: { type: "string" },+ snippet: { type: "string" },+ markdown: { type: "string" },+ },+ },+ },+ images: {+ type: "array",+ items: {+ type: "object",+ properties: {+ url: { type: "string" },+ title: { type: "string" },+ markdown: { type: "string" },+ },+ },+ },+ },+ },+ scrapeIds: {+ type: "object",+ description:+ "Job IDs for async scraping (if asyncScraping is true)",+ properties: {+ web: { type: "array", items: { type: "string" } },+ news: { type: "array", items: { type: "string" } },+ images: { type: "array", items: { type: "string" } },+ },+ },+ creditsUsed: { type: "number" },+ },+ },+ },+ },+ },+ facilitator,+ ),wrap(x402SearchController),);diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 228ef32b8..6f5371ddd 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -483,8 +483,12 @@ export class WebCrawler {}};+ let timeoutHandle: NodeJS.Timeout;const timeoutPromise = new Promise((_, reject) => {- setTimeout(() => reject(new Error("Sitemap fetch timeout")), timeout);+ timeoutHandle = setTimeout(+ () => reject(new Error("Sitemap fetch timeout")),+ timeout,+ );});// Allow sitemaps to be cached for 48 hours if they are requested from /map@@ -515,7 +519,9 @@ export class WebCrawler {),]).then(results => results.reduce((a, x) => a + x, 0)),timeoutPromise,- ])) as number;+ ]).finally(() => {+ clearTimeout(timeoutHandle);+ })) as number;if (count > 0) {if (diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 185e1c64c..a7f37f4a9 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -260,6 +260,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {} else {const file = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {headers: meta.options.headers,+ signal: meta.abort.asSignal(),});const ct = file.response.headers.get("Content-Type");@@ -295,6 +296,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }: await downloadFile(meta.id, meta.rewrittenUrl ?? meta.url, {headers: meta.options.headers,+ signal: meta.abort.asSignal(),});if ((response as any).headers) {diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.tsindex 831f40d90..21b14338b 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/downloadFile.ts@@ -48,15 +48,19 @@ export async function downloadFile(throw new EngineError("Response body was null", { cause: { response } });}- response.body.pipeTo(Writable.toWeb(tempFileWrite));- await new Promise((resolve, reject) => {- tempFileWrite.on("finish", () => resolve(null));- tempFileWrite.on("error", error => {- reject(- new EngineError("Failed to write to temp file", { cause: { error } }),- );- });- });+ try {+ await response.body+ .pipeTo(Writable.toWeb(tempFileWrite), {+ signal: init?.signal || undefined,+ })+ .catch(error => {+ throw new EngineError("Failed to write to temp file", {+ cause: { error },+ });+ });+ } finally {+ tempFileWrite.close();+ }return {response,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 2267f3b39..ad05a836f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -4,6 +4,7 @@ import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import {type Document,+ getPDFMaxPages,scrapeOptions,type ScrapeOptions,type TeamFlags,@@ -114,6 +115,7 @@ export type Meta = {| undefined; // undefined: no prefetch yet, null: prefetch came back emptycostTracking: CostTracking;winnerEngine?: Engine;+ abortHandle?: NodeJS.Timeout;};function buildFeatureFlags(@@ -278,6 +280,18 @@ async function buildMetaObject(crawlId: internalOptions.crawlId,});+ const abortController = new AbortController();+ const abortHandle =+ options.timeout !== undefined+ ? setTimeout(+ () =>+ abortController.abort(+ new ScrapeJobTimeoutError("Scrape timed out"),+ ),+ options.timeout,+ )+ : undefined;+return {id,url,@@ -293,11 +307,12 @@ async function buildMetaObject(},internalOptions,logger,+ abortHandle,abort: new AbortManager(internalOptions.externalAbort,options.timeout !== undefined? {- signal: AbortSignal.timeout(options.timeout),+ signal: abortController.signal,tier: "scrape",timesOutAt: new Date(Date.now() + options.timeout),throwable() {@@ -357,63 +372,79 @@ async function scrapeURLLoopIter(engine: Engine,snipeAbort,): Promise<EngineScrapeResult> {- const engineResult = await scrapeURLWithEngine(- {- ...meta,- abort: meta.abort.child(snipeAbort),- },- engine,- );-- let checkMarkdown = await parseMarkdown(- await htmlTransform(- engineResult.html,- meta.url,- scrapeOptions.parse({ onlyMainContent: true }),- ),- );-- if (checkMarkdown.trim().length === 0) {- checkMarkdown = await parseMarkdown(- await htmlTransform(- engineResult.html,- meta.url,- scrapeOptions.parse({ onlyMainContent: false }),- ),+ const abort = meta.abort.child(snipeAbort);+ try {+ const engineResult = await scrapeURLWithEngine(+ {+ ...meta,+ abort,+ },+ engine,);- }- // Success factors- const isLongEnough = checkMarkdown.trim().length > 0;- const isGoodStatusCode =- (engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||- engineResult.statusCode === 304;- const hasNoPageError = engineResult.error === undefined;- const isLikelyProxyError = [401, 403, 429].includes(engineResult.statusCode);+ let checkMarkdown: string;+ if (meta.internalOptions.teamId === "sitemap") {+ checkMarkdown = engineResult.html?.trim() ?? "";+ } else {+ checkMarkdown = await parseMarkdown(+ await htmlTransform(+ engineResult.html,+ meta.url,+ scrapeOptions.parse({ onlyMainContent: true }),+ ),+ );++ if (checkMarkdown.trim().length === 0) {+ checkMarkdown = await parseMarkdown(+ await htmlTransform(+ engineResult.html,+ meta.url,+ scrapeOptions.parse({ onlyMainContent: false }),+ ),+ );+ }+ }- if (- isLikelyProxyError &&- meta.options.proxy === "auto" &&- !meta.featureFlags.has("stealthProxy")- ) {- meta.logger.info(- "Scrape via " +- engine +- " deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.",+ // Success factors+ const isLongEnough = checkMarkdown.trim().length > 0;+ const isGoodStatusCode =+ (engineResult.statusCode >= 200 && engineResult.statusCode < 300) ||+ engineResult.statusCode === 304;+ const hasNoPageError = engineResult.error === undefined;+ const isLikelyProxyError = [401, 403, 429].includes(+ engineResult.statusCode,);- throw new AddFeatureError(["stealthProxy"]);- }- // NOTE: TODO: what to do when status code is bad is tough...- // we cannot just rely on text because error messages can be brief and not hit the limit- // should we just use all the fallbacks and pick the one with the longest text? - mogery- if (isLongEnough || !isGoodStatusCode) {- meta.logger.info("Scrape via " + engine + " deemed successful.", {- factors: { isLongEnough, isGoodStatusCode, hasNoPageError },- });- return engineResult;- } else {- throw new EngineUnsuccessfulError(engine);+ if (+ isLikelyProxyError &&+ meta.options.proxy === "auto" &&+ !meta.featureFlags.has("stealthProxy")+ ) {+ meta.logger.info(+ "Scrape via " ++ engine ++ " deemed unsuccessful due to proxy inadequacy. Adding stealthProxy flag.",+ );+ throw new AddFeatureError(["stealthProxy"]);+ }++ // NOTE: TODO: what to do when status code is bad is tough...+ // we cannot just rely on text because error messages can be brief and not hit the limit+ // should we just use all the fallbacks and pick the one with the longest text? - mogery+ if (isLongEnough || !isGoodStatusCode) {+ meta.logger.info("Scrape via " + engine + " deemed successful.", {+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },+ });+ return engineResult;+ } else {+ meta.logger.warn("Scrape via " + engine + " deemed unsuccessful.", {+ factors: { isLongEnough, isGoodStatusCode, hasNoPageError },+ length: engineResult.html?.trim().length ?? 0,+ });+ throw new EngineUnsuccessfulError(engine);+ }+ } finally {+ abort?.dispose();}}@@ -539,35 +570,40 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {});while (true) {+ let timeouts: NodeJS.Timeout[] = [];try {result = await Promise.race([...enginePromises.map(x => x.promise),...(remainingEngines.length > 0? [new Promise<EngineScrapeResultWithContext>((_, reject) => {- setTimeout(() => {- reject(new WaterfallNextEngineSignal());- }, waitUntilWaterfall);+ timeouts.push(+ setTimeout(() => {+ reject(new WaterfallNextEngineSignal());+ }, waitUntilWaterfall),+ );}),]: []),new Promise<EngineScrapeResultWithContext>((_, reject) => {- setTimeout(() => {- try {- meta.abort.throwIfAborted();-- // Fallback error if above doesn't throw- const usingDefaultTimeout =- meta.abort.scrapeTimeout() === undefined;- throw new ScrapeJobTimeoutError(- usingDefaultTimeout- ? "Scrape timed out due to maximum length of 5 minutes"- : "Scrape timed out",- );- } catch (error) {- reject(error);- }- }, meta.abort.scrapeTimeout() ?? 300000);+ timeouts.push(+ setTimeout(() => {+ try {+ meta.abort.throwIfAborted();++ // Fallback error if above doesn't throw+ const usingDefaultTimeout =+ meta.abort.scrapeTimeout() === undefined;+ throw new ScrapeJobTimeoutError(+ usingDefaultTimeout+ ? "Scrape timed out due to maximum length of 5 minutes"+ : "Scrape timed out",+ );+ } catch (error) {+ reject(error);+ }+ }, meta.abort.scrapeTimeout() ?? 300000),+ );}),]);break;@@ -650,6 +686,8 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {} else if (error instanceof WaterfallNextEngineSignal) {// It's time to waterfall the next enginebreak;+ } else if (error instanceof ScrapeJobTimeoutError) {+ throw error;} else if (error instanceof AbortManagerThrownError) {if (error.tier === "engine") {meta.logger.warn(@@ -665,6 +703,10 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {});throw error;}+ } finally {+ for (const to of timeouts) {+ clearTimeout(to);+ }}}@@ -775,6 +817,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {: warning;}+ // NOTE: for sitemap, we don't need all the transformers, need to skip unused onesdocument = await executeTransformers(meta, document);// Set final span attributes@@ -1075,6 +1118,28 @@ export async function scrapeURL(result.success && result.document.metadata.cacheState === "hit",});+ if (useIndex) {+ meta.logger.debug("scrapeURL index metrics", {+ module: "scrapeURL/index-metrics",+ timeTaken: Date.now() - startTime,+ changeTracking: hasFormatOfType(+ meta.options.formats,+ "changeTracking",+ ),+ branding: hasFormatOfType(meta.options.formats, "branding"),+ pdfMaxPages: getPDFMaxPages(meta.options.parsers),+ maxAge: meta.options.maxAge,+ headers: meta.options.headers+ ? Object.keys(meta.options.headers).length+ : 0,+ actions: meta.options.actions?.length ?? 0,+ proxy: meta.options.proxy,+ success: result.success,+ indexHit:+ result.success && result.document.metadata.cacheState === "hit",+ });+ }+setSpanAttributes(span, {"scrape.success": true,"scrape.duration_ms": Date.now() - startTime,@@ -1096,6 +1161,27 @@ export async function scrapeURL(indexHit: false,});+ if (useIndex) {+ meta.logger.debug("scrapeURL index metrics", {+ module: "scrapeURL/index-metrics",+ timeTaken: Date.now() - startTime,+ changeTracking: hasFormatOfType(+ meta.options.formats,+ "changeTracking",+ ),+ branding: hasFormatOfType(meta.options.formats, "branding"),+ pdfMaxPages: getPDFMaxPages(meta.options.parsers),+ maxAge: meta.options.maxAge,+ headers: meta.options.headers+ ? Object.keys(meta.options.headers).length+ : 0,+ actions: meta.options.actions?.length ?? 0,+ proxy: meta.options.proxy,+ success: false,+ indexHit: false,+ });+ }+// Set error attributes on spanlet errorType = "unknown";if (error instanceof NoEnginesLeftError) {diff --git a/apps/api/src/scraper/scrapeURL/lib/abortManager.ts b/apps/api/src/scraper/scrapeURL/lib/abortManager.tsindex be25f0711..70f7e275b 100644--- a/apps/api/src/scraper/scrapeURL/lib/abortManager.ts+++ b/apps/api/src/scraper/scrapeURL/lib/abortManager.ts@@ -8,6 +8,7 @@ export type AbortInstance = {export class AbortManager {private aborts: AbortInstance[] = [];private mappedController: AbortController | null = null;+ private listeners: Array<{ signal: AbortSignal; handler: () => void }> = [];constructor(...instances: (AbortInstance | undefined | null)[]) {this.aborts = instances.filter(@@ -15,6 +16,38 @@ export class AbortManager {) as AbortInstance[];}+ dispose() {+ for (const { signal, handler } of this.listeners) {+ signal.removeEventListener("abort", handler);+ }++ this.listeners = [];+ this.aborts = [];+ this.mappedController = null;+ }++ private resolveInner(abort: AbortInstance): any {+ try {+ return abort.throwable();+ } catch (err) {+ return err;+ }+ }++ private register(abort: AbortInstance) {+ const handler = () => {+ if (!this.mappedController) return;++ const inner = this.resolveInner(abort);+ const reason = new AbortManagerThrownError(abort.tier, inner);++ this.mappedController.abort(reason);+ };++ abort.signal.addEventListener("abort", handler);+ this.listeners.push({ signal: abort.signal, handler });+ }+add(...instances: (AbortInstance | undefined | null)[]) {const pureInstances = instances.filter(x => x !== undefined && x !== null,@@ -23,11 +56,7 @@ export class AbortManager {if (this.mappedController !== null) {for (const abort of pureInstances) {- abort.signal.addEventListener("abort", () => {- this.mappedController?.abort(- new AbortManagerThrownError(abort.tier, abort.throwable()),- );- });+ this.register(abort);}}}@@ -49,7 +78,8 @@ export class AbortManager {throwIfAborted(): void {for (const abort of this.aborts) {if (abort.signal.aborted) {- throw new AbortManagerThrownError(abort.tier, abort.throwable());+ const inner = this.resolveInner(abort);+ throw new AbortManagerThrownError(abort.tier, inner);}}}@@ -58,11 +88,7 @@ export class AbortManager {this.mappedController = new AbortController();for (const abort of this.aborts) {- abort.signal.addEventListener("abort", () => {- this.mappedController?.abort(- new AbortManagerThrownError(abort.tier, abort.throwable()),- );- });+ this.register(abort);}}diff --git a/apps/api/src/scraper/scrapeURL/lib/fetch.ts b/apps/api/src/scraper/scrapeURL/lib/fetch.tsindex da9866a05..6d87b0ffc 100644--- a/apps/api/src/scraper/scrapeURL/lib/fetch.ts+++ b/apps/api/src/scraper/scrapeURL/lib/fetch.ts@@ -220,9 +220,16 @@ export async function robustFetch<},);if (tryCooldown !== undefined) {- await new Promise(resolve =>- setTimeout(() => resolve(null), tryCooldown),- );+ let timeoutHandle: NodeJS.Timeout | null = null;+ try {+ await new Promise<null>(resolve => {+ timeoutHandle = setTimeout(() => resolve(null), tryCooldown);+ });+ } finally {+ if (timeoutHandle) {+ clearTimeout(timeoutHandle);+ }+ }}return await robustFetch({...params,diff --git a/apps/api/src/search/v2/ddgsearch.ts b/apps/api/src/search/v2/ddgsearch.tsindex 2200950c6..2f05cbb14 100644--- a/apps/api/src/search/v2/ddgsearch.ts+++ b/apps/api/src/search/v2/ddgsearch.ts@@ -134,7 +134,12 @@ export async function ddgSearch(let antiBotRetries = 0;while (results.length < num_results && nextPageData) {- const timeoutSignal = AbortSignal.timeout(timeout);+ const abortController = new AbortController();+ const timeoutHandle = setTimeout(() => {+ if (abortController) {+ abortController.abort();+ }+ }, timeout);try {let response: undici.Response;@@ -153,7 +158,7 @@ export async function ddgSearch("Accept-Encoding": "gzip, deflate, br","Upgrade-Insecure-Requests": "1",},- signal: timeoutSignal,+ signal: abortController.signal,},);} else {@@ -170,7 +175,7 @@ export async function ddgSearch("Accept-Encoding": "gzip, deflate, br","Upgrade-Insecure-Requests": "1",},- signal: timeoutSignal,+ signal: abortController.signal,});}@@ -204,7 +209,10 @@ export async function ddgSearch(} else {throw error;}+ } finally {+ if (timeoutHandle) clearTimeout(timeoutHandle);}+await new Promise(r => setTimeout(r, 1000));}diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.tsindex 97b252629..9be7a6057 100644--- a/apps/api/src/services/ab-test.ts+++ b/apps/api/src/services/ab-test.ts@@ -28,15 +28,21 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {webScraperOptions.internalOptions?.v1Agent === undefined &&webScraperOptions.internalOptions?.v1JSONAgent === undefined;if (shouldABTest) {+ let timeout = Math.min(+ 60000,+ (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000,+ );+(async () => {+ const abortController = new AbortController();+ const timeoutHandle = setTimeout(() => {+ if (abortController) {+ abortController.abort();+ }+ }, timeout);+try {abLogger.info("A/B-testing scrapeURL to staging");- const abort = AbortSignal.timeout(- Math.min(- 60000,- (webScraperOptions.scrapeOptions.timeout ?? 30000) + 10000,- ),- );await robustFetch({url: `http://${abHostEnv}/v2/scrape`,method: "POST",@@ -50,11 +56,13 @@ export function abTestJob(webScraperOptions: ScrapeJobData) {tryCount: 1,ignoreResponse: true,mock: null,- abort,+ abort: abortController.signal,});abLogger.info("A/B-testing scrapeURL (staging) request sent");} catch (error) {abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });+ } finally {+ if (timeoutHandle) clearTimeout(timeoutHandle);}})();}@@ -84,12 +92,18 @@ export function abTestFireEngine(Math.random() <= abRate &&abHostEnv;if (shouldABTest) {+ let timeout = Math.min(60000, (feRequest.timeout ?? 30000) + 10000);+(async () => {+ const abortController = new AbortController();+ const timeoutHandle = setTimeout(() => {+ if (abortController) {+ abortController.abort();+ }+ }, timeout);+try {abLogger.info("A/B-testing scrapeURL to staging");- const abort = AbortSignal.timeout(- Math.min(60000, (feRequest.timeout ?? 30000) + 10000),- );await robustFetch({url: `http://${abHostEnv}/scrape`,method: "POST",@@ -98,11 +112,13 @@ export function abTestFireEngine(tryCount: 1,ignoreResponse: true,mock: null,- abort,+ abort: abortController.signal,});abLogger.info("A/B-testing scrapeURL (staging) request sent");} catch (error) {abLogger.warn("A/B-testing scrapeURL (staging) failed", { error });+ } finally {+ if (timeoutHandle) clearTimeout(timeoutHandle);}})();}diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex dde434a34..6a009c15b 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -523,6 +523,7 @@ export async function waitForJob(const jobId = typeof job == "string" ? job : job.id;const isConcurrencyLimited = !!(typeof job === "string");+ let timeoutHandle: NodeJS.Timeout | null = null;let doc: Document | null = null;try {doc = await Promise.race(@@ -534,7 +535,7 @@ export async function waitForJob(),timeout !== null? new Promise<Document>((_resolve, reject) => {- setTimeout(() => {+ timeoutHandle = setTimeout(() => {reject(new ScrapeJobTimeoutError("Scrape timed out" +@@ -561,7 +562,12 @@ export async function waitForJob(} else {throw e;}+ } finally {+ if (timeoutHandle) {+ clearTimeout(timeoutHandle);+ }}+logger.debug("Got job");if (!doc) {diff --git a/apps/api/src/services/supabase.ts b/apps/api/src/services/supabase.tsindex 5eb11d1af..62a08baf1 100644--- a/apps/api/src/services/supabase.ts+++ b/apps/api/src/services/supabase.ts@@ -8,6 +8,8 @@ class SupabaseService {private client: SupabaseClient | null = null;private rrClient: SupabaseClient | null = null;+ private acucClient: SupabaseClient | null = null;+constructor() {const supabaseUrl = process.env.SUPABASE_URL;const supabaseReplicaUrl = process.env.SUPABASE_REPLICA_URL;@@ -15,6 +17,26 @@ class SupabaseService {const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";// Only initialize the Supabase client if both URL and Service Token are provided.if (!useDbAuthentication) {+ if (!!process.env.SUPABASE_ACUC_URL) {+ const supabaseAcucUrl = process.env.SUPABASE_ACUC_URL;+ const supabaseAcucServiceToken =+ process.env.SUPABASE_ACUC_SERVICE_TOKEN;+ if (!supabaseAcucUrl || !supabaseAcucServiceToken) {+ logger.error(+ "Supabase ACUC environment variables aren't configured correctly. Supabase ACUC client will not be initialized. Fix ENV configuration or disable ACUC with USE_DB_AUTHENTICATION env variable",+ );+ this.acucClient = null;+ } else {+ this.acucClient = createClient(+ supabaseAcucUrl,+ supabaseAcucServiceToken,+ );+ logger.info(+ "Supabase ACUC only client initialized, this is only for request authentication",+ );+ }+ }+// Warn the user that Authentication is disabled by setting the client to nulllogger.warn("Authentication is disabled. Supabase client will not be initialized.",@@ -45,6 +67,10 @@ class SupabaseService {getRRClient(): SupabaseClient | null {return this.rrClient;}++ getACUCOnlyClient(): SupabaseClient | null {+ return this.acucClient;+ }}const serv = new SupabaseService();@@ -86,3 +112,21 @@ export const supabase_rr_service: SupabaseClient = new Proxy(serv, {return Reflect.get(client, prop, receiver);},}) as unknown as SupabaseClient;++export const supabase_acuc_only_service: SupabaseClient = new Proxy(serv, {+ get: function (target, prop, receiver) {+ const client = target.getACUCOnlyClient();+ // If the Supabase client is not initialized, intercept property access to provide meaningful error feedback.+ if (client === null) {+ return () => {+ throw new Error("Supabase RR client is not configured.");+ };+ }+ // Direct access to SupabaseService properties takes precedence.+ if (prop in target) {+ return Reflect.get(target, prop, receiver);+ }+ // Otherwise, delegate access to the Supabase client.+ return Reflect.get(client, prop, receiver);+ },+}) as unknown as SupabaseClient;diff --git a/apps/api/src/services/webhook/delivery.ts b/apps/api/src/services/webhook/delivery.tsindex 4f89dd362..2f6b23662 100644--- a/apps/api/src/services/webhook/delivery.ts+++ b/apps/api/src/services/webhook/delivery.ts@@ -98,13 +98,23 @@ export class WebhookSender {headers["X-Firecrawl-Signature"] = `sha256=${hmac.digest("hex")}`;}+ const abortController = new AbortController();+ const timeoutHandle = setTimeout(+ () => {+ if (abortController) {+ abortController.abort();+ }+ },+ this.context.v0 ? 30000 : 10000,+ );+try {const res = await undici.fetch(this.config.url, {method: "POST",headers,body: payloadString,dispatcher: getSecureDispatcher(),- signal: AbortSignal.timeout(this.context.v0 ? 30000 : 10000),+ signal: abortController.signal,});if (!res.ok) {@@ -146,6 +156,8 @@ export class WebhookSender {});throw error;+ } finally {+ if (timeoutHandle) clearTimeout(timeoutHandle);}}diff --git a/apps/api/src/services/worker/redis.ts b/apps/api/src/services/worker/redis.tsnew file mode 100644index 000000000..bca1aa2a8--- /dev/null+++ b/apps/api/src/services/worker/redis.ts@@ -0,0 +1,118 @@+import IORedis, { Redis } from "ioredis";+import { logger as _logger } from "../../lib/logger";++const logger = _logger.child({ module: "nuq/redis" });++const luaScripts = {+ semaphore: {+ acquire: `+-- KEYS[1]=leases_zset ; ARGV[1]=holder_id, ARGV[2]=limit, ARGV[3]=lease_ttl_ms+local t = redis.call('TIME')+local now_ms = t[1]*1000 + math.floor(t[2]/1000)++local exp = redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms)++if redis.call('ZSCORE', KEYS[1], ARGV[1]) then+ return {1, exp, 0}+end++local in_use = tonumber(redis.call('ZCARD', KEYS[1]))++if in_use < tonumber(ARGV[2]) then+ redis.call('ZADD', KEYS[1], 'NX', now_ms + tonumber(ARGV[3]), ARGV[1])+ return {1, exp, in_use}+else+ local first = redis.call('ZRANGE', KEYS[1], 0, 0, 'WITHSCORES')+ return {0, exp, in_use}+end`,++ release: `+-- KEYS[1]=leases_zset ; ARGV[1]=holder_id+return redis.call('ZREM', KEYS[1], ARGV[1])`,++ heartbeat: `+-- KEYS[1]=leases_zset ; ARGV[1]=holder_id, ARGV[2]=lease_ttl_ms+local t = redis.call('TIME')+local now_ms = t[1] * 1000 + math.floor(t[2] / 1000)++redis.call('ZREMRANGEBYSCORE', KEYS[1], '-inf', now_ms)++local curr = redis.call('ZSCORE', KEYS[1], ARGV[1])+if not curr then+ return 0+end++local new_expiry = now_ms + tonumber(ARGV[2])+redis.call('ZADD', KEYS[1], 'XX', new_expiry, ARGV[1])++return 1`,+ },+} as const;++type ScriptHashes = {+ [K in keyof typeof luaScripts]: {+ [K2 in keyof (typeof luaScripts)[K]]: string;+ };+};++const scripts: ScriptHashes = {} as ScriptHashes;++const redis = new IORedis(process.env.REDIS_URL!, {+ lazyConnect: true,+ maxRetriesPerRequest: null,+ enableReadyCheck: false,+});++let initPromise: Promise<void> | null = null;++export const ensureRedis = async () => {+ if (initPromise) return initPromise;+ initPromise = (async () => {+ await redis.connect();++ for (const [k, v] of Object.entries(luaScripts)) {+ scripts[k] = {};+ for (const [k2, v2] of Object.entries(v)) {+ const h = await redis.script("LOAD", v2);+ scripts[k][k2] = h as string;+ }+ }+ logger.info("Redis connected and scripts loaded");+ })().catch(err => {+ initPromise = null;+ throw err;+ });+ return initPromise;+};++export const semaphoreKeys = (teamId: string) => {+ return {+ leases: `nuq:sema:{${teamId}}:leases`,+ };+};++async function runScript<T>(+ hash: string,+ keys: string[],+ args?: (string | number)[],+): Promise<T> {+ await ensureRedis();+ return (await redis.evalsha(+ hash,+ keys.length,+ ...keys,+ ...(args || []),+ )) as T;+}++type NuQRedis = Redis & {+ scripts: typeof scripts;+ runScript: typeof runScript;+ ensure: typeof ensureRedis;+};++export const nuqRedis: NuQRedis = Object.assign(redis, {+ scripts,+ runScript,+ ensure: ensureRedis,+});diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 79c3d5c79..1e5af10e8 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -71,8 +71,7 @@ import {withSpan,setSpanAttributes,} from "../../lib/otel-tracer";--const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));+import { ScrapeUrlResponse } from "../../scraper/scrapeURL";configDotenv();@@ -81,8 +80,10 @@ const jobLockExtendInterval =const jobLockExtensionTime =Number(process.env.JOB_LOCK_EXTENSION_TIME) || 60000;-cacheableLookup.install(http.globalAgent);-cacheableLookup.install(https.globalAgent);+if (require.main === module) {+ cacheableLookup.install(http.globalAgent);+ cacheableLookup.install(https.globalAgent);+}async function billScrapeJob(job: NuQJob<any>,@@ -169,13 +170,23 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const costTracking = new CostTracking();+ const abortController = new AbortController();+ const abortTimeoutHandle =+ remainingTime !== undefined+ ? setTimeout(+ () =>+ abortController.abort(+ new ScrapeJobTimeoutError("Scrape timed out"),+ ),+ remainingTime,+ )+ : undefined;+ const signal = abortController.signal;+try {if (remainingTime !== undefined && remainingTime < 0) {throw new ScrapeJobTimeoutError("Scrape timed out");}- const signal = remainingTime- ? AbortSignal.timeout(remainingTime)- : undefined;if (job.data.crawl_id) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;@@ -184,20 +195,31 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {}}- const pipeline = await Promise.race([- startWebScraperPipeline({- job,- costTracking,- }),- ...(remainingTime !== undefined- ? [- (async () => {- await sleep(remainingTime);- throw new ScrapeJobTimeoutError("Scrape timed out");- })(),- ]- : []),- ]);+ let pipeline: ScrapeUrlResponse | null = null;+ let timeoutHandle: NodeJS.Timeout | null = null;+ try {+ pipeline = await Promise.race([+ startWebScraperPipeline({+ job,+ costTracking,+ }),+ ...(remainingTime !== undefined+ ? [+ (async () => {+ await new Promise(resolve => {+ timeoutHandle = setTimeout(resolve, remainingTime);+ });++ throw new ScrapeJobTimeoutError("Scrape timed out");+ })(),+ ]+ : []),+ ]);+ } finally {+ if (timeoutHandle) {+ clearTimeout(timeoutHandle);+ }+ }try {signal?.throwIfAborted();@@ -692,6 +714,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {job.data.internalOptions?.bypassBilling ?? false,);return data;+ } finally {+ if (abortTimeoutHandle) clearTimeout(abortTimeoutHandle);}}@@ -1097,7 +1121,11 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {try {try {let extendLockInterval: NodeJS.Timeout | null = null;- if (job.data?.mode !== "kickoff" && job.data?.team_id) {+ if (+ job.data?.mode !== "kickoff" &&+ job.data?.team_id &&+ !job.data.skipNuq+ ) {extendLockInterval = setInterval(async () => {await pushConcurrencyLimitActiveJob(job.data.team_id,@@ -1144,7 +1172,7 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {}try {- if (process.env.GCS_BUCKET_NAME) {+ if (process.env.GCS_BUCKET_NAME && !job.data.skipNuq) {logger.debug("Job succeeded -- putting null in Redis");return null;} else {@@ -1163,15 +1191,22 @@ async function processJobWithTracing(job: NuQJob<ScrapeJobData>, logger: any) {}}} finally {- await concurrentJobDone(job);+ if (!job.data.skipNuq) {+ await concurrentJobDone(job);+ }}} catch (error) {logger.warn("Job failed", { error });Sentry.captureException(error);- if (error instanceof TransportableError) {- throw new Error(serializeTransportableError(error));++ if (job.data.skipNuq) {+ throw error;} else {- throw new Error(serializeTransportableError(new UnknownError(error)));+ if (error instanceof TransportableError) {+ throw new Error(serializeTransportableError(error));+ } else {+ throw new Error(serializeTransportableError(new UnknownError(error)));+ }}}}diff --git a/apps/api/src/services/worker/team-semaphore.ts b/apps/api/src/services/worker/team-semaphore.tsnew file mode 100644index 000000000..169c11470--- /dev/null+++ b/apps/api/src/services/worker/team-semaphore.ts@@ -0,0 +1,202 @@+import { isSelfHosted } from "../../lib/deployment";+import { ScrapeJobTimeoutError, TransportableError } from "../../lib/error";+import { logger as _logger } from "../../lib/logger";+import { nuqRedis, semaphoreKeys } from "./redis";+import { Gauge, Histogram, register } from "prom-client";++const activeSemaphores = new Gauge({+ name: "noq_semaphore_active",+ help: "Number of active semaphore holders",+});++const semaphoreAcquireDuration = new Histogram({+ name: "noq_semaphore_acquire_duration_seconds",+ help: "Semaphore acquire time",+ // Buckets in seconds: 5ms, 10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s+ buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],+});++const { scripts, runScript, ensure } = nuqRedis;++const SEMAPHORE_TTL = 30 * 1000;++async function acquire(+ teamId: string,+ holderId: string,+ limit: number,+): Promise<{ granted: boolean; count: number; removed: number }> {+ await ensure();++ const keys = semaphoreKeys(teamId);+ const [granted, count, removed] = await runScript<[number, number, number]>(+ scripts.semaphore.acquire,+ [keys.leases],+ [holderId, limit, SEMAPHORE_TTL],+ );++ return {+ granted: granted === 1,+ count,+ removed,+ };+}++async function acquireBlocking(+ teamId: string,+ holderId: string,+ limit: number,+ options: {+ base_delay_ms: number;+ max_delay_ms: number;+ timeout_ms: number;+ signal: AbortSignal;+ },+): Promise<{ limited: boolean; removed: number }> {+ await ensure();++ const deadline = Date.now() + options.timeout_ms;+ const keys = semaphoreKeys(teamId);++ let delay = options.base_delay_ms;+ let totalRemoved = 0;+ let failedOnce = false;++ // Start the timer for the histogram+ const endTimer = semaphoreAcquireDuration.startTimer();++ do {+ if (options.signal.aborted) {+ throw new ScrapeJobTimeoutError("Scrape timed out");+ }++ if (deadline < Date.now()) {+ throw new ScrapeJobTimeoutError("Scrape timed out");+ }++ const [granted, _count, _removed] = await runScript<+ [number, number, number]+ >(+ scripts.semaphore.acquire,+ [keys.leases],+ [holderId, limit, SEMAPHORE_TTL],+ );++ totalRemoved++;++ if (granted === 1) {+ // Stop the timer and record the duration+ endTimer();+ return { limited: failedOnce, removed: totalRemoved };+ }++ failedOnce = true;++ const jitter = Math.floor(+ Math.random() * Math.max(1, Math.floor(delay / 4)),+ );+ await new Promise(r => setTimeout(r, delay + jitter));++ delay = Math.min(options.max_delay_ms, Math.floor(delay * 1.5));+ } while (true);+}++async function heartbeat(teamId: string, holderId: string): Promise<boolean> {+ await ensure();++ const keys = semaphoreKeys(teamId);+ return (+ (await runScript<number>(+ scripts.semaphore.heartbeat,+ [keys.leases],+ [holderId, SEMAPHORE_TTL],+ )) === 1+ );+}++async function release(teamId: string, holderId: string): Promise<void> {+ await ensure();++ const keys = semaphoreKeys(teamId);+ await runScript<number>(scripts.semaphore.release, [keys.leases], [holderId]);+}++async function count(teamId: string): Promise<number> {+ await ensure();++ const keys = semaphoreKeys(teamId);+ const count = await nuqRedis.zcard(keys.leases);+ return count;+}++function startHeartbeat(teamId: string, holderId: string, intervalMs: number) {+ let stopped = false;++ const promise = (async () => {+ while (!stopped) {+ const ok = await heartbeat(teamId, holderId);+ if (!ok) {+ throw new TransportableError("SCRAPE_TIMEOUT", "heartbeat_failed");+ }+ await new Promise(r => setTimeout(r, intervalMs));+ }+ return Promise.reject(+ new Error("heartbeat loop stopped unexpectedly"),+ ) as never;+ })();++ return {+ promise,+ stop() {+ stopped = true;+ },+ };+}++async function withSemaphore<T>(+ teamId: string,+ holderId: string,+ limit: number,+ signal: AbortSignal,+ timeoutMs: number,+ func: (limited: boolean) => Promise<T>,+): Promise<T> {+ if (isSelfHosted() && limit <= 1) {+ _logger.debug(`Bypassing concurrency limit for ${teamId}`, {+ teamId,+ jobId: holderId,+ });+ return await func(false);+ }++ const { limited } = await acquireBlocking(teamId, holderId, limit, {+ base_delay_ms: 25,+ max_delay_ms: 250,+ timeout_ms: timeoutMs,+ signal,+ });++ const hb = startHeartbeat(teamId, holderId, SEMAPHORE_TTL / 2);++ activeSemaphores.inc();+ try {+ const result = await Promise.race([func(limited), hb.promise]);+ return result;+ } finally {+ activeSemaphores.dec();+ hb.stop();++ await release(teamId, holderId).catch(() => {});+ }+}++const getMetrics = async () => {+ return register.metrics();+};++export const teamConcurrencySemaphore = {+ acquire,+ release,+ withSemaphore,+ count,+ getMetrics,+};diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 86f96b6b9..11dd06418 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -17,6 +17,7 @@ type ScrapeJobCommon = {team_id: string;zeroDataRetention: boolean;traceContext?: SerializedTraceContext;+ skipNuq?: boolean;};export type ScrapeJobData = ScrapeJobCommon &diff --git a/apps/go-html-to-md-service/.dockerignore b/apps/go-html-to-md-service/.dockerignorenew file mode 100644index 000000000..fde6ac73c--- /dev/null+++ b/apps/go-html-to-md-service/.dockerignore@@ -0,0 +1,27 @@+# Ignore build artifacts+*.dylib+*.so+*.dll+*.exe++# Ignore test files+*_test.go++# Ignore documentation+README.md+*.md++# Ignore git+.git+.gitignore++# Ignore IDE files+.vscode+.idea+*.swp+*.swo++# Ignore temporary files+*.tmp+*.log+diff --git a/apps/go-html-to-md-service/.gitignore b/apps/go-html-to-md-service/.gitignorenew file mode 100644index 000000000..ea5bf9a40--- /dev/null+++ b/apps/go-html-to-md-service/.gitignore@@ -0,0 +1,31 @@+# Binaries for programs and plugins+*.exe+*.exe~+*.dll+*.so+*.dylib++# Test binary, built with `go test -c`+*.test++# Output of the go coverage tool+*.out++# Dependency directories+vendor/++# Go workspace file+go.work++# IDE specific files+.vscode/+.idea/+*.swp+*.swo++# Logs+*.log++# Binary+html-to-markdown-service+diff --git a/apps/go-html-to-md-service/Dockerfile b/apps/go-html-to-md-service/Dockerfilenew file mode 100644index 000000000..ab3909c5b--- /dev/null+++ b/apps/go-html-to-md-service/Dockerfile@@ -0,0 +1,42 @@+# Build stage+FROM golang:1.23-alpine AS builder++# Install build dependencies+RUN apk add --no-cache git++# Set working directory+WORKDIR /app++# Copy go mod files+COPY go.mod go.sum ./++# Download dependencies+RUN go mod download++# Copy source code+COPY . .++# Build the application+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o html-to-markdown-service .++# Final stage+FROM alpine:latest++# Install ca-certificates for HTTPS requests+RUN apk --no-cache add ca-certificates++WORKDIR /root/++# Copy the binary from builder+COPY --from=builder /app/html-to-markdown-service .++# Expose port+EXPOSE 8080++# Health check+HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \+ CMD wget --no-verbose --tries=1 --spider http://localhost:8080/health || exit 1++# Run the application+CMD ["./html-to-markdown-service"]+diff --git a/apps/go-html-to-md-service/Makefile b/apps/go-html-to-md-service/Makefilenew file mode 100644index 000000000..e234e23da--- /dev/null+++ b/apps/go-html-to-md-service/Makefile@@ -0,0 +1,87 @@+.PHONY: help build test run clean docker-build docker-run docker-stop install lint fmt++# Default target+help:+ @echo "HTML to Markdown Service - Available targets:"+ @echo " make build - Build the service binary"+ @echo " make test - Run tests"+ @echo " make test-cover - Run tests with coverage"+ @echo " make run - Run the service locally"+ @echo " make clean - Clean build artifacts"+ @echo " make install - Install dependencies"+ @echo " make lint - Run linter"+ @echo " make fmt - Format code"+ @echo " make docker-build - Build Docker image"+ @echo " make docker-run - Run with Docker Compose"+ @echo " make docker-stop - Stop Docker containers"+ @echo " make docker-logs - View Docker logs"++# Build the service+build:+ @echo "Building service..."+ go build -o html-to-markdown-service .++# Run tests+test:+ @echo "Running tests..."+ go test -v ./...++# Run tests with coverage+test-cover:+ @echo "Running tests with coverage..."+ go test -v -cover -coverprofile=coverage.out ./...+ go tool cover -html=coverage.out -o coverage.html+ @echo "Coverage report generated: coverage.html"++# Run the service+run:+ @echo "Starting service..."+ go run .++# Clean build artifacts+clean:+ @echo "Cleaning..."+ rm -f html-to-markdown-service+ rm -f coverage.out coverage.html+ go clean++# Install dependencies+install:+ @echo "Installing dependencies..."+ go mod download+ go mod tidy++# Run linter (requires golangci-lint)+lint:+ @echo "Running linter..."+ golangci-lint run++# Format code+fmt:+ @echo "Formatting code..."+ go fmt ./...++# Build Docker image+docker-build:+ @echo "Building Docker image..."+ docker build -t html-to-markdown-service:latest .++# Run with Docker Compose+docker-run:+ @echo "Starting with Docker Compose..."+ docker-compose up -d++# Stop Docker containers+docker-stop:+ @echo "Stopping Docker containers..."+ docker-compose down++# View Docker logs+docker-logs:+ @echo "Viewing Docker logs..."+ docker-compose logs -f++# Run all checks before commit+pre-commit: fmt lint test+ @echo "Pre-commit checks passed!"+diff --git a/apps/go-html-to-md-service/converter.go b/apps/go-html-to-md-service/converter.gonew file mode 100644index 000000000..0d09bc072--- /dev/null+++ b/apps/go-html-to-md-service/converter.go@@ -0,0 +1,145 @@+package main++import (+ "strings"+ "unicode/utf8"++ "github.com/PuerkitoBio/goquery"+ md "github.com/firecrawl/html-to-markdown"+ "github.com/firecrawl/html-to-markdown/plugin"+ "golang.org/x/net/html"+)++// Converter handles HTML to Markdown conversion+type Converter struct {+ converter *md.Converter+}++// NewConverter creates a new Converter instance with pre-configured rules+func NewConverter() *Converter {+ converter := md.NewConverter("", true, nil)+ converter.Use(plugin.GitHubFlavored())+ addGenericPreRule(converter)++ return &Converter{+ converter: converter,+ }+}++// ConvertHTMLToMarkdown converts HTML string to Markdown+func (c *Converter) ConvertHTMLToMarkdown(html string) (string, error) {+ return c.converter.ConvertString(html)+}++// addGenericPreRule adds a robust PRE handler that extracts nested code text+// (e.g., tables/rows/gutters) and outputs fenced blocks with detected language.+func addGenericPreRule(conv *md.Converter) {+ isGutter := func(class string) bool {+ c := strings.ToLower(class)+ return strings.Contains(c, "gutter") || strings.Contains(c, "line-numbers")+ }++ detectLang := func(sel *goquery.Selection) string {+ classes := sel.AttrOr("class", "")+ lower := strings.ToLower(classes)+ for _, part := range strings.Fields(lower) {+ if strings.HasPrefix(part, "language-") {+ return strings.TrimPrefix(part, "language-")+ }+ if strings.HasPrefix(part, "lang-") {+ return strings.TrimPrefix(part, "lang-")+ }+ }+ return ""+ }++ // Collect text recursively; insert newlines after block elements and br+ var collect func(n *html.Node, b *strings.Builder)+ collect = func(n *html.Node, b *strings.Builder) {+ if n == nil {+ return+ }+ switch n.Type {+ case html.TextNode:+ b.WriteString(n.Data)+ case html.ElementNode:+ name := strings.ToLower(n.Data)+ // Skip gutters+ if name != "" {+ // check class attr for gutters+ for _, a := range n.Attr {+ if a.Key == "class" && isGutter(a.Val) {+ return+ }+ }+ }++ if name == "br" {+ b.WriteString("\n")+ }++ for c := n.FirstChild; c != nil; c = c.NextSibling {+ collect(c, b)+ }++ // Newline after block-ish wrappers to preserve lines+ switch name {+ case "p", "div", "li", "tr", "table", "thead", "tbody", "tfoot", "section", "article", "blockquote", "pre", "h1", "h2", "h3", "h4", "h5", "h6":+ b.WriteString("\n")+ }+ }+ }++ conv.AddRules(md.Rule{+ Filter: []string{"pre"},+ Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {+ // find inner <code> if present for language+ codeSel := selec.Find("code").First()+ lang := detectLang(codeSel)+ if lang == "" {+ lang = detectLang(selec)+ }++ var b strings.Builder+ for _, n := range selec.Nodes {+ collect(n, &b)+ }+ content := strings.TrimRight(b.String(), "\n")++ fenceChar, _ := utf8.DecodeRuneInString(opt.Fence)+ fence := md.CalculateCodeFence(fenceChar, content)+ text := "\n\n" + fence + lang + "\n" + content + "\n" + fence + "\n\n"+ return md.String(text)+ },+ })++ // Inline code: robustly extract text and fence with backticks+ conv.AddRules(md.Rule{+ Filter: []string{"code"},+ Replacement: func(_ string, selec *goquery.Selection, opt *md.Options) *string {+ // If inside pre, let the PRE rule handle it+ if selec.ParentsFiltered("pre").Length() > 0 {+ return nil+ }+ var b strings.Builder+ for _, n := range selec.Nodes {+ collect(n, &b)+ }+ code := b.String()+ // collapse multiple newlines for inline code+ code = md.TrimTrailingSpaces(strings.ReplaceAll(code, "\r\n", "\n"))++ // Choose fence length safely+ fence := "`"+ if strings.Contains(code, "`") {+ fence = "``"+ if strings.Contains(code, "``") {+ fence = "```"+ }+ }+ out := fence + code + fence+ return md.String(out)+ },+ })+}+diff --git a/apps/go-html-to-md-service/docker-compose.yml b/apps/go-html-to-md-service/docker-compose.ymlnew file mode 100644index 000000000..36ca2caec--- /dev/null+++ b/apps/go-html-to-md-service/docker-compose.yml@@ -0,0 +1,20 @@+version: '3.8'++services:+ html-to-markdown:+ build:+ context: .+ dockerfile: Dockerfile+ container_name: html-to-markdown-service+ ports:+ - "8080:8080"+ environment:+ - PORT=8080+ restart: unless-stopped+ healthcheck:+ test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8080/health"]+ interval: 30s+ timeout: 3s+ retries: 3+ start_period: 5s+diff --git a/apps/go-html-to-md-service/go.mod b/apps/go-html-to-md-service/go.modnew file mode 100644index 000000000..1fdf8e997--- /dev/null+++ b/apps/go-html-to-md-service/go.mod@@ -0,0 +1,23 @@+module github.com/firecrawl/go-html-to-md-service++go 1.23.0++require (+ github.com/PuerkitoBio/goquery v1.10.3+ github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3+ github.com/gorilla/mux v1.8.1+ github.com/rs/zerolog v1.33.0+ golang.org/x/net v0.41.0+)++require (+ github.com/andybalholm/cascadia v1.3.3 // indirect+ github.com/kr/pretty v0.3.0 // indirect+ github.com/mattn/go-colorable v0.1.13 // indirect+ github.com/mattn/go-isatty v0.0.20 // indirect+ golang.org/x/sys v0.33.0 // indirect+ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect+ gopkg.in/yaml.v2 v2.4.0 // indirect+)++replace github.com/JohannesKaufmann/html-to-markdown => github.com/firecrawl/html-to-markdown v0.0.0-20250917145228-b6d0a75dfdbadiff --git a/apps/go-html-to-md-service/go.sum b/apps/go-html-to-md-service/go.sumnew file mode 100644index 000000000..e10a7c0c1--- /dev/null+++ b/apps/go-html-to-md-service/go.sum@@ -0,0 +1,117 @@+github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=+github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=+github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3 h1:lzHIpN3DszdL8V2JRK03WleWIeW2ssVmiMAbg67ES/A=+github.com/firecrawl/html-to-markdown v0.0.0-20250922154302-32a7ad4a22c3/go.mod h1:jngam+MdNp7FZkhSTlFsuA5hXY21X0+vuiGlpgo2n5o=+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=+github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=+github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=+github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y=+github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=+github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=+github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=+github.com/yuin/goldmark v1.7.1 h1:3bajkSilaCbjdKVsKdZjZCLBNPL9pYzrCakKaf4U49U=+github.com/yuin/goldmark v1.7.1/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=diff --git a/apps/go-html-to-md-service/handler.go b/apps/go-html-to-md-service/handler.gonew file mode 100644index 000000000..f995bc54b--- /dev/null+++ b/apps/go-html-to-md-service/handler.go@@ -0,0 +1,164 @@+package main++import (+ "encoding/json"+ "io"+ "net/http"+ "time"++ "github.com/gorilla/mux"+ "github.com/rs/zerolog/log"+)++const (+ maxRequestSize = 10 * 1024 * 1024 // 10MB max request size+)++// Handler manages HTTP request handling+type Handler struct {+ converter *Converter+}++// NewHandler creates a new Handler instance+func NewHandler(converter *Converter) *Handler {+ return &Handler{+ converter: converter,+ }+}++// RegisterRoutes registers all HTTP routes+func (h *Handler) RegisterRoutes(router *mux.Router) {+ router.HandleFunc("/health", h.HealthCheck).Methods("GET")+ router.HandleFunc("/convert", h.ConvertHTML).Methods("POST")+ router.HandleFunc("/", h.Index).Methods("GET")+}++// HealthCheckResponse represents the health check response+type HealthCheckResponse struct {+ Status string `json:"status"`+ Timestamp time.Time `json:"timestamp"`+ Service string `json:"service"`+}++// HealthCheck handles health check requests+func (h *Handler) HealthCheck(w http.ResponseWriter, r *http.Request) {+ response := HealthCheckResponse{+ Status: "healthy",+ Timestamp: time.Now(),+ Service: "html-to-markdown",+ }++ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ json.NewEncoder(w).Encode(response)+}++// IndexResponse represents the index page response+type IndexResponse struct {+ Service string `json:"service"`+ Version string `json:"version"`+ Description string `json:"description"`+ Endpoints []string `json:"endpoints"`+}++// Index handles root path requests+func (h *Handler) Index(w http.ResponseWriter, r *http.Request) {+ response := IndexResponse{+ Service: "HTML to Markdown Converter",+ Version: "1.0.0",+ Description: "A service for converting HTML content to Markdown format",+ Endpoints: []string{+ "GET /health - Health check endpoint",+ "POST /convert - Convert HTML to Markdown",+ },+ }++ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ json.NewEncoder(w).Encode(response)+}++// ConvertRequest represents the conversion request payload+type ConvertRequest struct {+ HTML string `json:"html"`+}++// ConvertResponse represents the conversion response payload+type ConvertResponse struct {+ Markdown string `json:"markdown"`+ Success bool `json:"success"`+}++// ErrorResponse represents an error response+type ErrorResponse struct {+ Error string `json:"error"`+ Success bool `json:"success"`+}++// ConvertHTML handles HTML to Markdown conversion requests+func (h *Handler) ConvertHTML(w http.ResponseWriter, r *http.Request) {+ startTime := time.Now()++ // Limit request body size+ r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)++ // Read and decode request body+ body, err := io.ReadAll(r.Body)+ if err != nil {+ log.Error().Err(err).Msg("Failed to read request body")+ h.sendError(w, "Failed to read request body", http.StatusBadRequest)+ return+ }++ var req ConvertRequest+ if err := json.Unmarshal(body, &req); err != nil {+ log.Error().Err(err).Msg("Failed to parse request body")+ h.sendError(w, "Invalid JSON in request body", http.StatusBadRequest)+ return+ }++ // Validate input+ if req.HTML == "" {+ h.sendError(w, "HTML field is required", http.StatusBadRequest)+ return+ }++ // Convert HTML to Markdown+ markdown, err := h.converter.ConvertHTMLToMarkdown(req.HTML)+ if err != nil {+ log.Error().Err(err).Msg("Failed to convert HTML to Markdown")+ h.sendError(w, "Failed to convert HTML to Markdown", http.StatusInternalServerError)+ return+ }++ // Log metrics+ duration := time.Since(startTime)+ log.Info().+ Dur("duration_ms", duration).+ Int("input_size", len(req.HTML)).+ Int("output_size", len(markdown)).+ Msg("HTML to Markdown conversion completed")++ // Send response+ response := ConvertResponse{+ Markdown: markdown,+ Success: true,+ }++ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusOK)+ json.NewEncoder(w).Encode(response)+}++// sendError sends an error response+func (h *Handler) sendError(w http.ResponseWriter, message string, statusCode int) {+ response := ErrorResponse{+ Error: message,+ Success: false,+ }++ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(statusCode)+ json.NewEncoder(w).Encode(response)+}+diff --git a/apps/go-html-to-md-service/handler_test.go b/apps/go-html-to-md-service/handler_test.gonew file mode 100644index 000000000..7ea308682--- /dev/null+++ b/apps/go-html-to-md-service/handler_test.go@@ -0,0 +1,261 @@+package main++import (+ "bytes"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "testing"++ "github.com/gorilla/mux"+)++func TestHealthCheck(t *testing.T) {+ converter := NewConverter()+ handler := NewHandler(converter)++ router := mux.NewRouter()+ handler.RegisterRoutes(router)++ req, err := http.NewRequest("GET", "/health", nil)+ if err != nil {+ t.Fatal(err)+ }++ rr := httptest.NewRecorder()+ router.ServeHTTP(rr, req)++ if status := rr.Code; status != http.StatusOK {+ t.Errorf("handler returned wrong status code: got %v want %v",+ status, http.StatusOK)+ }++ var response HealthCheckResponse+ if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {+ t.Errorf("failed to parse response: %v", err)+ }++ if response.Status != "healthy" {+ t.Errorf("handler returned unexpected status: got %v want %v",+ response.Status, "healthy")+ }++ if response.Service != "html-to-markdown" {+ t.Errorf("handler returned unexpected service: got %v want %v",+ response.Service, "html-to-markdown")+ }+}++func TestIndex(t *testing.T) {+ converter := NewConverter()+ handler := NewHandler(converter)++ router := mux.NewRouter()+ handler.RegisterRoutes(router)++ req, err := http.NewRequest("GET", "/", nil)+ if err != nil {+ t.Fatal(err)+ }++ rr := httptest.NewRecorder()+ router.ServeHTTP(rr, req)++ if status := rr.Code; status != http.StatusOK {+ t.Errorf("handler returned wrong status code: got %v want %v",+ status, http.StatusOK)+ }++ var response IndexResponse+ if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {+ t.Errorf("failed to parse response: %v", err)+ }++ if response.Service != "HTML to Markdown Converter" {+ t.Errorf("handler returned unexpected service name")+ }++ if len(response.Endpoints) == 0 {+ t.Errorf("handler returned no endpoints")+ }+}++func TestConvertHTML_Success(t *testing.T) {+ converter := NewConverter()+ handler := NewHandler(converter)++ router := mux.NewRouter()+ handler.RegisterRoutes(router)++ testCases := []struct {+ name string+ html string+ expectedOutput string+ }{+ {+ name: "Simple paragraph",+ html: "<p>Hello, World!</p>",+ expectedOutput: "Hello, World!",+ },+ {+ name: "Bold text",+ html: "<p>This is <strong>bold</strong> text</p>",+ expectedOutput: "**bold**",+ },+ {+ name: "Link",+ html: "<a href='https://example.com'>Example</a>",+ expectedOutput: "[Example](https://example.com)",+ },+ {+ name: "Code block",+ html: "<pre><code>console.log('hello');</code></pre>",+ },+ {+ name: "Inline code",+ html: "<code>const x = 1;</code>",+ expectedOutput: "`const x = 1;`",+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ reqBody := ConvertRequest{+ HTML: tc.html,+ }+ jsonBody, _ := json.Marshal(reqBody)++ req, err := http.NewRequest("POST", "/convert", bytes.NewBuffer(jsonBody))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")++ rr := httptest.NewRecorder()+ router.ServeHTTP(rr, req)++ if status := rr.Code; status != http.StatusOK {+ t.Errorf("handler returned wrong status code: got %v want %v",+ status, http.StatusOK)+ }++ var response ConvertResponse+ if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {+ t.Errorf("failed to parse response: %v", err)+ }++ if !response.Success {+ t.Errorf("conversion was not successful")+ }++ if tc.expectedOutput != "" && response.Markdown == "" {+ t.Errorf("expected markdown output, got empty string")+ }++ // For simple checks, verify expected output is contained in response+ if tc.expectedOutput != "" {+ if !contains(response.Markdown, tc.expectedOutput) {+ t.Errorf("expected markdown to contain %q, got %q",+ tc.expectedOutput, response.Markdown)+ }+ }+ })+ }+}++func TestConvertHTML_EmptyHTML(t *testing.T) {+ converter := NewConverter()+ handler := NewHandler(converter)++ router := mux.NewRouter()+ handler.RegisterRoutes(router)++ reqBody := ConvertRequest{+ HTML: "",+ }+ jsonBody, _ := json.Marshal(reqBody)++ req, err := http.NewRequest("POST", "/convert", bytes.NewBuffer(jsonBody))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")++ rr := httptest.NewRecorder()+ router.ServeHTTP(rr, req)++ if status := rr.Code; status != http.StatusBadRequest {+ t.Errorf("handler returned wrong status code: got %v want %v",+ status, http.StatusBadRequest)+ }++ var response ErrorResponse+ if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil {+ t.Errorf("failed to parse response: %v", err)+ }++ if response.Success {+ t.Errorf("expected success to be false")+ }+}++func TestConvertHTML_InvalidJSON(t *testing.T) {+ converter := NewConverter()+ handler := NewHandler(converter)++ router := mux.NewRouter()+ handler.RegisterRoutes(router)++ req, err := http.NewRequest("POST", "/convert", bytes.NewBuffer([]byte("invalid json")))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")++ rr := httptest.NewRecorder()+ router.ServeHTTP(rr, req)++ if status := rr.Code; status != http.StatusBadRequest {+ t.Errorf("handler returned wrong status code: got %v want %v",+ status, http.StatusBadRequest)+ }+}++func TestConverter_ComplexHTML(t *testing.T) {+ converter := NewConverter()++ testHTML := `+ <div>+ <h1>Title</h1>+ <p>This is a paragraph with <strong>bold</strong> and <em>italic</em> text.</p>+ <ul>+ <li>Item 1</li>+ <li>Item 2</li>+ </ul>+ <pre><code class="language-javascript">console.log('hello');</code></pre>+ </div>+ `++ markdown, err := converter.ConvertHTMLToMarkdown(testHTML)+ if err != nil {+ t.Errorf("conversion failed: %v", err)+ }++ if markdown == "" {+ t.Errorf("expected non-empty markdown output")+ }++ // Verify key elements are present+ expectedElements := []string{"Title", "bold", "italic", "Item 1", "console.log"}+ for _, elem := range expectedElements {+ if !contains(markdown, elem) {+ t.Errorf("expected markdown to contain %q, but it didn't", elem)+ }+ }+}++// Helper function to check if a string contains a substring+func contains(s, substr string) bool {+ return bytes.Contains([]byte(s), []byte(substr))+}+diff --git a/apps/go-html-to-md-service/main.go b/apps/go-html-to-md-service/main.gonew file mode 100644index 000000000..1da6e217d--- /dev/null+++ b/apps/go-html-to-md-service/main.go@@ -0,0 +1,100 @@+package main++import (+ "context"+ "net/http"+ "os"+ "os/signal"+ "syscall"+ "time"++ "github.com/gorilla/mux"+ "github.com/rs/zerolog"+ "github.com/rs/zerolog/log"+)++const (+ defaultPort = "8080"+ defaultShutdownTimeout = 30 * time.Second+ defaultReadTimeout = 30 * time.Second+ defaultWriteTimeout = 30 * time.Second+ maxUploadSize = 50 * 1024 * 1024+)++func main() {+ // Configure logging+ zerolog.TimeFieldFormat = zerolog.TimeFormatUnix++ env := os.Getenv("ENV")++ if env == "production" {+ zerolog.SetGlobalLevel(zerolog.InfoLevel)+ } else {+ log.Logger = log.Output(zerolog.ConsoleWriter{+ Out: os.Stdout,+ TimeFormat: time.RFC3339,+ })+ }++ // Get port from environment or use default+ port := os.Getenv("PORT")+ if port == "" {+ port = defaultPort+ }++ // Initialize converter+ converter := NewConverter()++ // Initialize handlers+ handler := NewHandler(converter)++ // Setup router+ router := mux.NewRouter()++ router.Use(func(next http.Handler) http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ r.Body = http.MaxBytesReader(w, r.Body, maxUploadSize)+ next.ServeHTTP(w, r)+ })+ })++ handler.RegisterRoutes(router)++ // Create server+ srv := &http.Server{+ Addr: ":" + port,+ Handler: router,+ ReadTimeout: defaultReadTimeout,+ WriteTimeout: defaultWriteTimeout,+ }++ // Start server in goroutine+ go func() {+ log.Info().+ Str("port", port).+ Str("env", env).+ Msg("Starting HTML to Markdown service")++ if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {+ log.Fatal().Err(err).Msg("Failed to start server")+ }+ }()++ // Wait for interrupt signal to gracefully shutdown the server+ quit := make(chan os.Signal, 1)+ signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM)+ <-quit++ log.Info().Msg("Shutting down server...")++ // Create shutdown context with timeout+ ctx, cancel := context.WithTimeout(context.Background(), defaultShutdownTimeout)+ defer cancel()++ // Attempt graceful shutdown+ if err := srv.Shutdown(ctx); err != nil {+ log.Fatal().Err(err).Msg("Server forced to shutdown")+ }++ log.Info().Msg("Server exited")+}diff --git a/apps/go-html-to-md-service/requests.http b/apps/go-html-to-md-service/requests.httpnew file mode 100644index 000000000..6ac797d9a--- /dev/null+++ b/apps/go-html-to-md-service/requests.http@@ -0,0 +1,224 @@+### HTML to Markdown Service - API Tests+### Use the REST Client extension in VS Code or IntelliJ to run these requests++@baseUrl = http://localhost:8080++### Health Check+GET {{baseUrl}}/health++###++### Service Info+GET {{baseUrl}}/++###++### Test 1: Simple HTML Conversion+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<h1>Hello World</h1>"+}++###++### Test 2: Paragraph with Bold and Italic+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<p>This is a <strong>bold</strong> and <em>italic</em> test.</p>"+}++###++### Test 3: Unordered List+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<ul><li>First item</li><li>Second item</li><li>Third item</li></ul>"+}++###++### Test 4: Ordered List+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<ol><li>Step 1</li><li>Step 2</li><li>Step 3</li></ol>"+}++###++### Test 5: Code Block with Language+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<pre><code class=\"language-javascript\">function hello() {\n console.log('Hello, World!');\n}</code></pre>"+}++###++### Test 6: Inline Code+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<p>Use the <code>console.log()</code> function to debug.</p>"+}++###++### Test 7: Links and Anchors+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<p>Visit <a href=\"https://example.com\">our website</a> for more information.</p>"+}++###++### Test 8: Images+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<img src=\"https://example.com/image.jpg\" alt=\"Example Image\">"+}++###++### Test 9: Table+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<table><thead><tr><th>Name</th><th>Age</th><th>City</th></tr></thead><tbody><tr><td>John</td><td>30</td><td>New York</td></tr><tr><td>Jane</td><td>25</td><td>London</td></tr></tbody></table>"+}++###++### Test 10: Blockquote+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<blockquote><p>This is a blockquote.</p></blockquote>"+}++###++### Test 11: Horizontal Rule+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<p>Before</p><hr><p>After</p>"+}++###++### Test 12: Nested Structure+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<div><h1>Title</h1><p>Paragraph with <strong>bold</strong> text.</p><ul><li>Item 1</li><li>Item 2</li></ul></div>"+}++###++### Test 13: Complex Document+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<article><header><h1>Article Title</h1><p class=\"meta\">By Author Name</p></header><section><h2>Introduction</h2><p>This is the introduction with <strong>bold</strong> and <em>italic</em> text.</p><pre><code class=\"language-python\">def hello():\n print('Hello, World!')</code></pre></section><section><h2>Conclusion</h2><p>Visit <a href=\"https://example.com\">example.com</a> for more.</p></section></article>"+}++###++### Test 14: Strikethrough (GitHub Flavored Markdown)+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<p>This is <del>deleted</del> text.</p>"+}++###++### Test 15: Task List (GitHub Flavored Markdown)+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<ul><li><input type=\"checkbox\" checked> Completed task</li><li><input type=\"checkbox\"> Incomplete task</li></ul>"+}++###++### Test 16: Code Block with Multiple Languages+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<div><pre><code class=\"language-javascript\">console.log('JS');</code></pre><pre><code class=\"language-python\">print('Python')</code></pre><pre><code class=\"language-go\">fmt.Println(\"Go\")</code></pre></div>"+}++###++### Test 17: Mixed Content+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<div><h2>Tutorial</h2><p>First, install the package:</p><pre><code class=\"language-bash\">npm install package-name</code></pre><p>Then use it in your code:</p><pre><code class=\"language-javascript\">const pkg = require('package-name');</code></pre><p>For more info, see <a href=\"https://docs.example.com\">the docs</a>.</p></div>"+}++###++### Error Test 1: Empty HTML+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": ""+}++###++### Error Test 2: Invalid JSON+POST {{baseUrl}}/convert+Content-Type: application/json++{invalid json}++###++### Error Test 3: Missing HTML Field+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "notHtml": "<p>Test</p>"+}++###++### Performance Test: Large HTML (1000 paragraphs)+# Warning: This may take a few seconds+POST {{baseUrl}}/convert+Content-Type: application/json++{+ "html": "<div><h1>Large Document</h1><p>Paragraph 1</p><p>Paragraph 2</p><p>Paragraph 3</p><p>Paragraph 4</p><p>Paragraph 5</p><p>Paragraph 6</p><p>Paragraph 7</p><p>Paragraph 8</p><p>Paragraph 9</p><p>Paragraph 10</p><h2>Section</h2><p>More content here with <strong>bold</strong> and <em>italic</em> text.</p><ul><li>Item 1</li><li>Item 2</li><li>Item 3</li></ul><pre><code class=\"language-javascript\">console.log('code');</code></pre></div>"+}++###+