Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
engpicker POC (#2501)
Gergő Móricz085aab71373b
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex d2e122d7d..3172bbb8f 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -22,6 +22,7 @@ regex = "1.11.2"roxmltree = "0.20.0"serde = { version = "1.0.219", features = ["derive"] }serde_json = "1.0.143"+strsim = "0.11"texting_robots = "0.2.2"url = "2.5.7"zip = "5.0.0"diff --git a/apps/api/native/src/engpicker.rs b/apps/api/native/src/engpicker.rsnew file mode 100644index 000000000..b030eda7b--- /dev/null+++ b/apps/api/native/src/engpicker.rs@@ -0,0 +1,206 @@+use napi_derive::napi;+use serde::{Deserialize, Serialize};+use strsim::levenshtein;+use tokio::task;++/// Result of evaluating a single URL across different engines+#[derive(Deserialize, Serialize)]+#[napi(object)]+pub struct EngpickerUrlResult {+ pub url: String,+ pub cdp_basic_markdown: Option<String>,+ pub cdp_basic_success: bool,+ pub cdp_stealth_markdown: Option<String>,+ pub cdp_stealth_success: bool,+ pub tls_basic_markdown: Option<String>,+ pub tls_basic_success: bool,+ pub tls_stealth_markdown: Option<String>,+ pub tls_stealth_success: bool,+}++/// Verdict for a single URL+#[derive(Serialize)]+#[napi(object)]+pub struct EngpickerUrlVerdict {+ pub url: String,+ pub tls_client_sufficient: bool,+ pub cdp_failed: bool,+ pub similarity: Option<f64>,+ pub reason: String,+}++/// Final verdict enum+#[derive(Serialize)]+#[napi(string_enum)]+pub enum EngpickerFinalVerdict {+ /// tlsclient is sufficient for this site+ TlsClientOk,+ /// Chrome CDP is required for proper rendering+ ChromeCdpRequired,+ /// Too many CDP failures to determine verdict+ Uncertain,+}++/// Final verdict result+#[derive(Serialize)]+#[napi(object)]+pub struct EngpickerVerdict {+ pub url_verdicts: Vec<EngpickerUrlVerdict>,+ pub tls_client_ok_count: u32,+ pub chrome_cdp_required_count: u32,+ pub cdp_failed_count: u32,+ pub total_urls: u32,+ pub verdict: EngpickerFinalVerdict,+}++/// Compute engpicker verdict using Levenshtein distance to compare tlsclient vs chrome-cdp results.+///+/// Chrome-CDP is the gold standard. We compare tlsclient markdown against it to determine+/// if tlsclient is sufficient for scraping this site (i.e., JS rendering not required).+///+/// Arguments:+/// - results: scrape results for each URL+/// - similarity_threshold: minimum similarity (0.0-1.0) for tlsclient to be considered sufficient+/// - success_rate_threshold: minimum ratio of successful comparisons for a definitive verdict+/// - cdp_failure_threshold: maximum ratio of CDP failures before verdict becomes uncertain+#[napi]+pub async fn compute_engpicker_verdict(+ results: Vec<EngpickerUrlResult>,+ similarity_threshold: f64,+ success_rate_threshold: f64,+ cdp_failure_threshold: f64,+) -> napi::Result<EngpickerVerdict> {+ task::spawn_blocking(move || {+ _compute_engpicker_verdict(results, similarity_threshold, success_rate_threshold, cdp_failure_threshold)+ })+ .await+ .map_err(|e| {+ napi::Error::new(+ napi::Status::GenericFailure,+ format!("compute_engpicker_verdict join error: {e}"),+ )+ })?+}++fn _compute_engpicker_verdict(+ results: Vec<EngpickerUrlResult>,+ similarity_threshold: f64,+ success_rate_threshold: f64,+ cdp_failure_threshold: f64,+) -> napi::Result<EngpickerVerdict> {+ let url_verdicts: Vec<EngpickerUrlVerdict> = results+ .iter()+ .map(|result| {+ // Get the best chrome-cdp result as gold standard (prefer stealth if both succeeded)+ let gold_standard = if result.cdp_stealth_success && result.cdp_stealth_markdown.is_some() {+ result.cdp_stealth_markdown.as_ref()+ } else if result.cdp_basic_success && result.cdp_basic_markdown.is_some() {+ result.cdp_basic_markdown.as_ref()+ } else {+ None+ };++ // Get the best tlsclient result (prefer stealth if both succeeded)+ let tls_result = if result.tls_stealth_success && result.tls_stealth_markdown.is_some() {+ result.tls_stealth_markdown.as_ref()+ } else if result.tls_basic_success && result.tls_basic_markdown.is_some() {+ result.tls_basic_markdown.as_ref()+ } else {+ None+ };++ // If chrome-cdp failed, we can't evaluate this URL+ let gold_standard = match gold_standard {+ Some(gs) if !gs.is_empty() => gs,+ _ => {+ return EngpickerUrlVerdict {+ url: result.url.clone(),+ tls_client_sufficient: false,+ cdp_failed: true,+ similarity: None,+ reason: "chrome-cdp failed".to_string(),+ };+ }+ };++ // If tlsclient failed entirely, it's definitely not enough+ let tls_result = match tls_result {+ Some(tls) if !tls.is_empty() => tls,+ _ => {+ return EngpickerUrlVerdict {+ url: result.url.clone(),+ tls_client_sufficient: false,+ cdp_failed: false,+ similarity: None,+ reason: "tlsclient failed".to_string(),+ };+ }+ };++ // Calculate Levenshtein distance and normalize to similarity score+ let distance = levenshtein(gold_standard, tls_result);+ let max_length = gold_standard.len().max(tls_result.len());+ let similarity = if max_length > 0 {+ 1.0 - (distance as f64 / max_length as f64)+ } else {+ 1.0+ };++ let tls_client_sufficient = similarity >= similarity_threshold;++ let reason = if tls_client_sufficient {+ format!("{:.1}% similar - tlsclient captures full content", similarity * 100.0)+ } else {+ format!("{:.1}% similar - JS rendering likely required", similarity * 100.0)+ };++ EngpickerUrlVerdict {+ url: result.url.clone(),+ tls_client_sufficient,+ cdp_failed: false,+ similarity: Some(similarity),+ reason,+ }+ })+ .collect();++ let total_urls = url_verdicts.len() as u32;+ let cdp_failed_count = url_verdicts.iter().filter(|v| v.cdp_failed).count() as u32;+ let tls_client_ok_count = url_verdicts.iter().filter(|v| v.tls_client_sufficient).count() as u32;+ let chrome_cdp_required_count = url_verdicts.iter().filter(|v| !v.tls_client_sufficient && !v.cdp_failed).count() as u32;++ // Determine final verdict+ let verdict = if total_urls == 0 {+ EngpickerFinalVerdict::Uncertain+ } else {+ let cdp_failure_rate = cdp_failed_count as f64 / total_urls as f64;++ // If too many CDP failures, we can't make a confident verdict+ if cdp_failure_rate > cdp_failure_threshold {+ EngpickerFinalVerdict::Uncertain+ } else {+ // Calculate success rate among URLs where we could actually compare+ let comparable_urls = total_urls - cdp_failed_count;+ if comparable_urls == 0 {+ EngpickerFinalVerdict::Uncertain+ } else {+ let tls_ok_rate = tls_client_ok_count as f64 / comparable_urls as f64;+ if tls_ok_rate >= success_rate_threshold {+ EngpickerFinalVerdict::TlsClientOk+ } else {+ EngpickerFinalVerdict::ChromeCdpRequired+ }+ }+ }+ };++ Ok(EngpickerVerdict {+ url_verdicts,+ tls_client_ok_count,+ chrome_cdp_required_count,+ cdp_failed_count,+ total_urls,+ verdict,+ })+}+diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rsindex f5f5c48e7..731c4f2b4 100644--- a/apps/api/native/src/lib.rs+++ b/apps/api/native/src/lib.rs@@ -1,6 +1,7 @@#![deny(clippy::all)]pub use crate::crawler::*;+pub use crate::engpicker::*;pub use crate::html::*;pub use crate::pdf::*;pub use crate::utils::*;@@ -9,6 +10,7 @@ pub use crate::document::{DocumentConverter, DocumentType};mod crawler;mod document;+mod engpicker;mod html;mod pdf;mod utils;diff --git a/apps/api/src/lib/engpicker.ts b/apps/api/src/lib/engpicker.tsnew file mode 100644index 000000000..b517a1899--- /dev/null+++ b/apps/api/src/lib/engpicker.ts@@ -0,0 +1,297 @@+import { generateObject } from "ai";+import { z } from "zod";+import { scrapeOptions } from "../controllers/v2/types";+import { scrapeURL } from "../scraper/scrapeURL";+import type { Engine } from "../scraper/scrapeURL/engines";+import { index_supabase_service } from "../services";+import { CostTracking } from "./cost-tracking";+import { getModel } from "./generic-ai";+import { logger as _logger } from "./logger";+import type { Logger } from "winston";+import { computeEngpickerVerdict, type EngpickerUrlResult } from "../../native";++type EngpickerJob = {+ id: number;+ domain_hash: string;+ domain_level: number;+ picked_up_at: string | null;+ done: boolean;+ created_at: string;+};++async function evaluateURL(+ id: string,+ url: string,+ engine: Engine,+ stealth: boolean,+ logger: Logger,+): Promise<{+ engine: Engine;+ stealth: boolean;+ markdown: string | null;+ result: boolean;+}> {+ const scrapeResult = await scrapeURL(+ id,+ url,+ scrapeOptions.parse({+ proxy: stealth ? "stealth" : "basic",+ maxAge: 0,+ storeInCache: false,+ }),+ {+ forceEngine: engine,+ teamId: "engpicker",+ },+ new CostTracking(),+ );++ if (!scrapeResult.success) {+ logger.warn("Scrape failed", { scrapeResult });+ return {+ engine,+ stealth,+ markdown: null,+ result: false,+ };+ }++ const markdown = scrapeResult.document.markdown ?? "";++ logger.info("Scrape completed, waiting for AI evaluation");++ // Use GPT-4o-mini to evaluate if the scrape was actually successful+ const evaluationResult = await generateObject({+ model: getModel("gpt-4o-mini", "openai"),+ schema: z.object({+ is_successful: z.boolean(),+ }),+ messages: [+ {+ role: "system",+ content: `You are a web scraping quality evaluator. Your job is to determine if a web scrape was successful based on the returned markdown content.++A scrape should be considered UNSUCCESSFUL if the content indicates any of the following:+- Antibot/captcha challenges (e.g., Cloudflare, reCAPTCHA, hCaptcha, bot detection messages)+- Region/geo blocks (e.g., "not available in your region", "access denied from your location")+- HTTP error pages (4xx or 5xx errors like "404 Not Found", "403 Forbidden", "500 Internal Server Error")+- Access denied or authentication required pages+- Rate limiting messages+- Empty or near-empty content that suggests the page didn't load properly++A scrape should be considered SUCCESSFUL if:+- The content appears to be the actual page content with meaningful text+- The page loaded properly with real content visible++A scrape may still be successful if:+- Cookie consent walls block the actual content+- Paywalls block the actual content`,+ },+ {+ role: "user",+ content: `Evaluate if this scraped markdown content represents a successful scrape:\n\n${markdown.slice(0, 4000)}`,+ },+ ],+ });++ const isSuccess = evaluationResult.object.is_successful;++ logger.info("AI evaluation completed", { isSuccess });++ return {+ engine,+ stealth,+ markdown,+ result: isSuccess,+ };+}++export async function processEngpickerJob() {+ let logger = _logger.child({+ module: "engpicker",+ method: "processEngpickerJob",+ });++ const { data: jobData, error: getJobError } = await index_supabase_service+ .from("engpicker_queue")+ .update({+ picked_up_at: new Date().toISOString(),+ })+ .is("picked_up_at", null)+ .order("created_at", { ascending: false })+ .limit(1)+ .select("*");++ if (getJobError) {+ logger.error("Error picking up engpicker job", { getJobError });+ await new Promise(resolve => setTimeout(resolve, 1000));+ return;+ }++ const job: EngpickerJob | undefined = jobData?.[0] ?? undefined;+ if (!job) {+ logger.debug("No engpicker job to pick up");+ await new Promise(resolve => setTimeout(resolve, 1000));+ return;+ }++ logger = logger.child({+ jobId: job.id,+ });++ logger.info("Picked up engpicker job");++ // main shit here++ const { data: indexRows, error: indexRowsError } =+ await index_supabase_service+ .rpc("query_index_at_domain_split_level_omce", {+ i_level: job.domain_level,+ i_domain_hash: job.domain_hash,+ i_newer_than: new Date(+ new Date(job.created_at).valueOf() - 1000 * 60 * 60 * 24,+ ).toISOString(),+ })+ .limit(100);++ if (indexRowsError || !indexRows) {+ logger.error("Error querying index rows", { indexRowsError });+ await new Promise(resolve => setTimeout(resolve, 1000));+ return;+ }++ const randomizedURLs: string[] = [];+ for (let i = 0; i < 10; i++) {+ const elem = indexRows.splice(+ Math.floor(Math.random() * indexRows.length),+ 1,+ );+ if (elem && elem.length > 0) {+ randomizedURLs.push(elem[0].url);+ }+ }++ logger.info("Picked randomized URLs", { randomizedURLs });++ const results = await Promise.all(+ randomizedURLs.map(async url => ({+ url,+ results: await Promise.all([+ evaluateURL(+ job.id + "-cdp-basic",+ url,+ "fire-engine;chrome-cdp",+ false,+ logger.child({+ method: "evaluateURL",+ engine: "fire-engine;chrome-cdp",+ stealth: false,+ }),+ ),+ evaluateURL(+ job.id + "-cdp-stealth",+ url,+ "fire-engine;chrome-cdp;stealth",+ true,+ logger.child({+ method: "evaluateURL",+ engine: "fire-engine;chrome-cdp;stealth",+ stealth: true,+ }),+ ),+ evaluateURL(+ job.id + "-tlsclient-basic",+ url,+ "fire-engine;tlsclient",+ false,+ logger.child({+ method: "evaluateURL",+ engine: "fire-engine;tlsclient",+ stealth: false,+ }),+ ),+ evaluateURL(+ job.id + "-tlsclient-stealth",+ url,+ "fire-engine;tlsclient;stealth",+ true,+ logger.child({+ method: "evaluateURL",+ engine: "fire-engine;tlsclient;stealth",+ stealth: true,+ }),+ ),+ ]),+ })),+ );++ // Transform results into format for native Levenshtein comparison+ const nativeInput: EngpickerUrlResult[] = results.map(result => {+ const cdpBasic = result.results.find(+ r => r.engine === "fire-engine;chrome-cdp",+ );+ const cdpStealth = result.results.find(+ r => r.engine === "fire-engine;chrome-cdp;stealth",+ );+ const tlsBasic = result.results.find(+ r => r.engine === "fire-engine;tlsclient",+ );+ const tlsStealth = result.results.find(+ r => r.engine === "fire-engine;tlsclient;stealth",+ );++ return {+ url: result.url,+ cdpBasicMarkdown: cdpBasic?.markdown ?? undefined,+ cdpBasicSuccess: cdpBasic?.result ?? false,+ cdpStealthMarkdown: cdpStealth?.markdown ?? undefined,+ cdpStealthSuccess: cdpStealth?.result ?? false,+ tlsBasicMarkdown: tlsBasic?.markdown ?? undefined,+ tlsBasicSuccess: tlsBasic?.result ?? false,+ tlsStealthMarkdown: tlsStealth?.markdown ?? undefined,+ tlsStealthSuccess: tlsStealth?.result ?? false,+ };+ });++ // Use native Rust implementation for fast Levenshtein comparison+ const SIMILARITY_THRESHOLD = 0.85; // 85% similarity means tlsclient is good enough+ const SUCCESS_RATE_THRESHOLD = 0.7; // 70% of comparable URLs must pass for tlsclient to be OK+ const CDP_FAILURE_THRESHOLD = 0.5; // If more than 50% of CDP scrapes failed, verdict is uncertain+ const verdictResult = await computeEngpickerVerdict(+ nativeInput,+ SIMILARITY_THRESHOLD,+ SUCCESS_RATE_THRESHOLD,+ CDP_FAILURE_THRESHOLD,+ );++ // This is the verdict - "TlsClientOk", "ChromeCdpRequired", or "Uncertain"+ const verdict = verdictResult.verdict;++ const { error: insertVerdictError } = await index_supabase_service+ .from("engpicker_verdicts")+ .insert({+ domain_hash: job.domain_hash,+ verdict,+ });++ if (insertVerdictError) {+ logger.error("Error inserting engpicker verdict", { insertVerdictError });+ await new Promise(resolve => setTimeout(resolve, 1000));+ return;+ }++ const { error: updateJobError } = await index_supabase_service+ .from("engpicker_queue")+ .update({+ done: true,+ })+ .eq("id", job.id);++ if (updateJobError) {+ logger.error("Error updating engpicker job", { updateJobError });+ await new Promise(resolve => setTimeout(resolve, 1000));+ return;+ }++ logger.info("Engpicker job completed", { verdict });+}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 514668b02..ab58050eb 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -40,6 +40,7 @@ import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import { crawlGroup } from "../worker/nuq";import { getACUCTeam } from "../../controllers/auth";import { supabase_service } from "../supabase";+import { processEngpickerJob } from "../../lib/engpicker";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;const workerStalledCheckInterval =@@ -773,6 +774,17 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;5 * 60 * 1000,);+ const engpickerPromise = (async () => {+ while (!isShuttingDown) {+ await new Promise(resolve => setTimeout(resolve, 1000));+ try {+ await processEngpickerJob();+ } catch (e) {+ logger.error("Error processing engpicker job", { error: e });+ }+ }+ })();+// Search indexing is now handled by separate search service// The search service has its own worker that processes the queue// This worker no longer needs to process search index jobs@@ -795,7 +807,11 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;}// Wait for all workers to complete (which should only happen on shutdown)- await Promise.all([billingWorkerPromise, precrawlWorkerPromise]);+ await Promise.all([+ billingWorkerPromise,+ precrawlWorkerPromise,+ engpickerPromise,+ ]);clearInterval(indexInserterInterval);clearInterval(webhookInserterInterval);