Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(html-to-md): respect zero data retention in conversion service (#3435)
Abimael Martell96868c38a349
diff --git a/.github/workflows/test-go-html-to-md-service.yml b/.github/workflows/test-go-html-to-md-service.ymlnew file mode 100644index 000000000..14f694e60--- /dev/null+++ b/.github/workflows/test-go-html-to-md-service.yml@@ -0,0 +1,44 @@+name: Go HTML-to-Markdown Service Check++on:+ pull_request:+ branches:+ - main+ paths:+ - apps/go-html-to-md-service/**+ - .github/workflows/test-go-html-to-md-service.yml+ push:+ branches:+ - main+ paths:+ - apps/go-html-to-md-service/**+ workflow_dispatch:++jobs:+ build-vet-test:+ name: Build, Vet and Test+ runs-on: blacksmith-4vcpu-ubuntu-2404+ if: >-+ github.event_name == 'workflow_dispatch' ||+ github.event_name == 'pull_request' ||+ (github.event_name == 'push' && github.ref == 'refs/heads/main')+ steps:+ - uses: actions/checkout@v4++ - name: Set up Go+ uses: actions/setup-go@v5+ with:+ go-version: "1.23"+ cache-dependency-path: apps/go-html-to-md-service/go.sum++ - name: Build+ working-directory: ./apps/go-html-to-md-service+ run: go build ./...++ - name: Vet+ working-directory: ./apps/go-html-to-md-service+ run: go vet ./...++ - name: Test+ working-directory: ./apps/go-html-to-md-service+ run: go test ./...diff --git a/apps/api/src/lib/html-to-markdown-client.ts b/apps/api/src/lib/html-to-markdown-client.tsindex adf348001..bdaf12cd0 100644--- a/apps/api/src/lib/html-to-markdown-client.ts+++ b/apps/api/src/lib/html-to-markdown-client.ts@@ -38,6 +38,7 @@ export async function convertHTMLToMarkdownWithHttpService(context?: {logger?: Logger;requestId?: string;+ zeroDataRetention?: boolean;},): Promise<string> {if (!html || html.trim() === "") {@@ -46,6 +47,7 @@ export async function convertHTMLToMarkdownWithHttpService(const contextLogger = context?.logger || logger;const requestId = context?.requestId;+ const zeroDataRetention = context?.zeroDataRetention === true;const url = config.HTML_TO_MARKDOWN_SERVICE_URL;const startTime = Date.now();@@ -56,11 +58,16 @@ export async function convertHTMLToMarkdownWithHttpService("Content-Type": "application/json",};- // Add request ID header if available- if (requestId) {+ // Add request ID header if available, but never for ZDR — it would+ // let the downstream service correlate logs back to a customer scrape.+ if (requestId && !zeroDataRetention) {headers["X-Request-ID"] = requestId;}+ if (zeroDataRetention) {+ headers["X-Zero-Data-Retention"] = "true";+ }+const response = await axios.post<ConvertResponse>(`${url}/convert`,request,@@ -76,12 +83,14 @@ export async function convertHTMLToMarkdownWithHttpService(throw new Error("Conversion was not successful");}- contextLogger.debug("HTML to Markdown conversion successful", {- duration_ms: duration,- input_size: html.length,- output_size: response.data.markdown.length,- ...(requestId ? { request_id: requestId } : {}),- });+ if (!zeroDataRetention) {+ contextLogger.debug("HTML to Markdown conversion successful", {+ duration_ms: duration,+ input_size: html.length,+ output_size: response.data.markdown.length,+ ...(requestId ? { request_id: requestId } : {}),+ });+ }return response.data.markdown;} catch (error) {@@ -103,18 +112,19 @@ export async function convertHTMLToMarkdownWithHttpService(serviceUrl: url,});- // Capture in Sentry with additional context+ // Capture in Sentry with additional context (omit identifying fields+ // and content sizes when ZDR so nothing is retained in Sentry either).Sentry.captureException(error, {tags: {service: "html-to-markdown",status_code: statusCode,- ...(requestId ? { request_id: requestId } : {}),+ ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),},extra: {serviceUrl: url,errorMessage,errorDetails,- inputSize: html.length,+ ...(zeroDataRetention ? {} : { inputSize: html.length }),},});@@ -133,7 +143,7 @@ export async function convertHTMLToMarkdownWithHttpService();Sentry.captureException(error, {tags: {- ...(requestId ? { request_id: requestId } : {}),+ ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),},});throw error;diff --git a/apps/api/src/lib/html-to-markdown.ts b/apps/api/src/lib/html-to-markdown.tsindex 69bfa6941..1b217f3b5 100644--- a/apps/api/src/lib/html-to-markdown.ts+++ b/apps/api/src/lib/html-to-markdown.ts@@ -56,6 +56,7 @@ export async function parseMarkdown(context?: {logger?: Logger;requestId?: string;+ zeroDataRetention?: boolean;},): Promise<string> {if (!html) {@@ -64,6 +65,7 @@ export async function parseMarkdown(const contextLogger = context?.logger || logger;const requestId = context?.requestId;+ const zeroDataRetention = context?.zeroDataRetention === true;// Try HTTP service first if enabledif (config.HTML_TO_MARKDOWN_SERVICE_URL) {@@ -71,6 +73,7 @@ export async function parseMarkdown(let markdownContent = await convertHTMLToMarkdownWithHttpService(html, {logger: contextLogger,requestId,+ zeroDataRetention,});markdownContent = await postProcessMarkdown(markdownContent);return markdownContent;@@ -82,7 +85,7 @@ export async function parseMarkdown(Sentry.captureException(error, {tags: {fallback: "original_parser",- ...(requestId ? { request_id: requestId } : {}),+ ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),},});}@@ -102,7 +105,7 @@ export async function parseMarkdown() {Sentry.captureException(error, {tags: {- ...(requestId ? { request_id: requestId } : {}),+ ...(requestId && !zeroDataRetention ? { request_id: requestId } : {}),},});contextLogger.error(diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex ada28140b..4c48fc99e 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -533,13 +533,14 @@ async function scrapeURLLoopIter(checkMarkdown = engineResult.html?.trim() ?? "";} else {const requestId = meta.id || meta.internalOptions.crawlId;+ const zeroDataRetention = meta.internalOptions.zeroDataRetention;checkMarkdown = await parseMarkdown(await htmlTransform(engineResult.html,meta.url,scrapeOptions.parse({ onlyMainContent: true }),),- { logger: meta.logger, requestId },+ { logger: meta.logger, requestId, zeroDataRetention },);if (checkMarkdown.trim().length === 0) {@@ -549,7 +550,7 @@ async function scrapeURLLoopIter(meta.url,scrapeOptions.parse({ onlyMainContent: false }),),- { logger: meta.logger, requestId },+ { logger: meta.logger, requestId, zeroDataRetention },);}}diff --git a/apps/api/src/scraper/scrapeURL/transformers/agent.ts b/apps/api/src/scraper/scrapeURL/transformers/agent.tsindex ab120ec7e..a7c22f5d3 100644--- a/apps/api/src/scraper/scrapeURL/transformers/agent.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/agent.ts@@ -61,6 +61,7 @@ export async function performAgent(const markdown = await parseMarkdown(html, {logger: meta.logger,requestId,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,});document.markdown = markdown;}diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex 06c1c6a74..45fd933d7 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -128,6 +128,7 @@ async function deriveMarkdownFromHTML(document.markdown = await parseMarkdown(document.html, {logger: meta.logger,requestId,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,});if (@@ -150,6 +151,7 @@ async function deriveMarkdownFromHTML(document.markdown = await parseMarkdown(document.html, {logger: meta.logger,requestId,+ zeroDataRetention: meta.internalOptions.zeroDataRetention,});meta.logger.info("Fallback to full content extraction completed", {diff --git a/apps/go-html-to-md-service/handler.go b/apps/go-html-to-md-service/handler.goindex 795821658..8dc517dd6 100644--- a/apps/go-html-to-md-service/handler.go+++ b/apps/go-html-to-md-service/handler.go@@ -4,6 +4,7 @@ import ("encoding/json""io""net/http"+ "strings""time""github.com/gorilla/mux"@@ -100,10 +101,14 @@ type ErrorResponse struct {func (h *Handler) ConvertHTML(w http.ResponseWriter, r *http.Request) {startTime := time.Now()+ // Zero data retention requests must not retain any per-request log+ // entries that could correlate back to a customer scrape.+ zdr := strings.EqualFold(r.Header.Get("X-Zero-Data-Retention"), "true")+// Extract request ID from header for loggingrequestID := r.Header.Get("X-Request-ID")logger := log.Logger- if requestID != "" {+ if requestID != "" && !zdr {logger = log.With().Str("request_id", requestID).Logger()}@@ -140,13 +145,16 @@ func (h *Handler) ConvertHTML(w http.ResponseWriter, r *http.Request) {return}- // Log metrics- duration := time.Since(startTime)- logger.Info().- Dur("duration_ms", duration).- Int("input_size", len(req.HTML)).- Int("output_size", len(markdown)).- Msg("HTML to Markdown conversion completed")+ // Skip per-request success logs for ZDR requests so no record is kept+ // that ties a request_id or its input/output sizes to the customer.+ if !zdr {+ duration := time.Since(startTime)+ logger.Info().+ Dur("duration_ms", duration).+ Int("input_size", len(req.HTML)).+ Int("output_size", len(markdown)).+ Msg("HTML to Markdown conversion completed")+ }// Send responseresponse := ConvertResponse{diff --git a/apps/go-html-to-md-service/handler_test.go b/apps/go-html-to-md-service/handler_test.goindex 7ea308682..e559f28b1 100644--- a/apps/go-html-to-md-service/handler_test.go+++ b/apps/go-html-to-md-service/handler_test.go@@ -8,6 +8,8 @@ import ("testing""github.com/gorilla/mux"+ "github.com/rs/zerolog"+ "github.com/rs/zerolog/log")func TestHealthCheck(t *testing.T) {@@ -259,3 +261,83 @@ func contains(s, substr string) bool {return bytes.Contains([]byte(s), []byte(substr))}+func TestConvertHTML_ZeroDataRetention_SuppressesLogs(t *testing.T) {+ converter := NewConverter()+ handler := NewHandler(converter)++ router := mux.NewRouter()+ handler.RegisterRoutes(router)++ originalLogger := log.Logger+ t.Cleanup(func() {+ log.Logger = originalLogger+ })++ var logBuf bytes.Buffer+ log.Logger = zerolog.New(&logBuf).Level(zerolog.DebugLevel)++ reqBody := ConvertRequest{HTML: "<p>secret content</p>"}+ jsonBody, _ := json.Marshal(reqBody)++ req, err := http.NewRequest("POST", "/convert", bytes.NewBuffer(jsonBody))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("X-Request-ID", "scrape-id-should-not-be-logged")+ req.Header.Set("X-Zero-Data-Retention", "true")++ rr := httptest.NewRecorder()+ router.ServeHTTP(rr, req)++ if status := rr.Code; status != http.StatusOK {+ t.Fatalf("handler returned wrong status code: got %v want %v", status, http.StatusOK)+ }++ logs := logBuf.String()+ if logs != "" {+ t.Errorf("expected no logs for ZDR conversion, got: %q", logs)+ }+}++func TestConvertHTML_NonZDR_LogsRequestID(t *testing.T) {+ converter := NewConverter()+ handler := NewHandler(converter)++ router := mux.NewRouter()+ handler.RegisterRoutes(router)++ originalLogger := log.Logger+ t.Cleanup(func() {+ log.Logger = originalLogger+ })++ var logBuf bytes.Buffer+ log.Logger = zerolog.New(&logBuf).Level(zerolog.DebugLevel)++ reqBody := ConvertRequest{HTML: "<p>hello</p>"}+ jsonBody, _ := json.Marshal(reqBody)++ req, err := http.NewRequest("POST", "/convert", bytes.NewBuffer(jsonBody))+ if err != nil {+ t.Fatal(err)+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("X-Request-ID", "scrape-id-123")++ rr := httptest.NewRecorder()+ router.ServeHTTP(rr, req)++ if status := rr.Code; status != http.StatusOK {+ t.Fatalf("handler returned wrong status code: got %v want %v", status, http.StatusOK)+ }++ logs := logBuf.String()+ if !contains(logs, "scrape-id-123") {+ t.Errorf("expected request_id in logs for non-ZDR conversion, got: %q", logs)+ }+ if !contains(logs, "HTML to Markdown conversion completed") {+ t.Errorf("expected completion log for non-ZDR conversion, got: %q", logs)+ }+}+