Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: word jaccard diff (#2720)
tom443df62190fd
diff --git a/apps/api/src/services/ab-test-comparison.ts b/apps/api/src/services/ab-test-comparison.tsindex d2c209507..6c923a2f7 100644--- a/apps/api/src/services/ab-test-comparison.ts+++ b/apps/api/src/services/ab-test-comparison.ts@@ -2,7 +2,6 @@ import { Logger } from "winston";import { parseMarkdown } from "../lib/html-to-markdown";const AB_LOG_PREFIX = "[FE_AB_COMPARE]";-const MAX_CONTENT_SIZE = 1_000_000; // 1 MBconst VARIANCE_THRESHOLD = 0.05; // 5% allowed varianceexport interface FireEngineResponse {@@ -20,15 +19,16 @@ function calculateSimilarity(a: string, b: string): number {if (a === b) return 1;if (a.length === 0 || b.length === 0) return 0;- const maxLen = Math.max(a.length, b.length);- const minLen = Math.min(a.length, b.length);+ const wordsA = new Set(a.toLowerCase().split(/\s+/).filter(Boolean));+ const wordsB = new Set(b.toLowerCase().split(/\s+/).filter(Boolean));- let matches = 0;- for (let i = 0; i < minLen; i++) {- if (a[i] === b[i]) matches++;+ let intersection = 0;+ for (const w of wordsA) {+ if (wordsB.has(w)) intersection++;}- return matches / maxLen;+ const union = wordsA.size + wordsB.size - intersection;+ return intersection / union;}export function scheduleABComparison(@@ -57,14 +57,6 @@ export function scheduleABComparison(return;}- // Skip comparison if content too large- if (- productionResponse.content.length > MAX_CONTENT_SIZE ||- mirrorResult.response.content.length > MAX_CONTENT_SIZE- ) {- return;- }-// Convert HTML to markdown before comparingconst [prodMarkdown, mirrorMarkdown] = await Promise.all([parseMarkdown(productionResponse.content, { logger: abLogger }),