Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

engpicker POC (#2501)

Gergő Móricz085aab71373b
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
index d2e122d7d..3172bbb8f 100644
--- a/apps/api/native/Cargo.toml
+++ b/apps/api/native/Cargo.toml
@@ -22,6 +22,7 @@ regex = "1.11.2"
roxmltree = "0.20.0"
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.143"
+strsim = "0.11"
texting_robots = "0.2.2"
url = "2.5.7"
zip = "5.0.0"
diff --git a/apps/api/native/src/engpicker.rs b/apps/api/native/src/engpicker.rs
new file mode 100644
index 000000000..b030eda7b
--- /dev/null
+++ b/apps/api/native/src/engpicker.rs
@@ -0,0 +1,206 @@
+use napi_derive::napi;
+use serde::{Deserialize, Serialize};
+use strsim::levenshtein;
+use tokio::task;
+
+/// Result of evaluating a single URL across different engines
+#[derive(Deserialize, Serialize)]
+#[napi(object)]
+pub struct EngpickerUrlResult {
+ pub url: String,
+ pub cdp_basic_markdown: Option<String>,
+ pub cdp_basic_success: bool,
+ pub cdp_stealth_markdown: Option<String>,
+ pub cdp_stealth_success: bool,
+ pub tls_basic_markdown: Option<String>,
+ pub tls_basic_success: bool,
+ pub tls_stealth_markdown: Option<String>,
+ pub tls_stealth_success: bool,
+}
+
+/// Verdict for a single URL
+#[derive(Serialize)]
+#[napi(object)]
+pub struct EngpickerUrlVerdict {
+ pub url: String,
+ pub tls_client_sufficient: bool,
+ pub cdp_failed: bool,
+ pub similarity: Option<f64>,
+ pub reason: String,
+}
+
+/// Final verdict enum
+#[derive(Serialize)]
+#[napi(string_enum)]
+pub enum EngpickerFinalVerdict {
+ /// tlsclient is sufficient for this site
+ TlsClientOk,
+ /// Chrome CDP is required for proper rendering
+ ChromeCdpRequired,
+ /// Too many CDP failures to determine verdict
+ Uncertain,
+}
+
+/// Final verdict result
+#[derive(Serialize)]
+#[napi(object)]
+pub struct EngpickerVerdict {
+ pub url_verdicts: Vec<EngpickerUrlVerdict>,
+ pub tls_client_ok_count: u32,
+ pub chrome_cdp_required_count: u32,
+ pub cdp_failed_count: u32,
+ pub total_urls: u32,
+ pub verdict: EngpickerFinalVerdict,
+}
+
+/// Compute engpicker verdict using Levenshtein distance to compare tlsclient vs chrome-cdp results.
+///
+/// Chrome-CDP is the gold standard. We compare tlsclient markdown against it to determine
+/// if tlsclient is sufficient for scraping this site (i.e., JS rendering not required).
+///
+/// Arguments:
+/// - results: scrape results for each URL
+/// - similarity_threshold: minimum similarity (0.0-1.0) for tlsclient to be considered sufficient
+/// - success_rate_threshold: minimum ratio of successful comparisons for a definitive verdict
+/// - cdp_failure_threshold: maximum ratio of CDP failures before verdict becomes uncertain
+#[napi]
+pub async fn compute_engpicker_verdict(
+ results: Vec<EngpickerUrlResult>,
+ similarity_threshold: f64,
+ success_rate_threshold: f64,
+ cdp_failure_threshold: f64,
+) -> napi::Result<EngpickerVerdict> {
+ task::spawn_blocking(move || {
+ _compute_engpicker_verdict(results, similarity_threshold, success_rate_threshold, cdp_failure_threshold)
+ })
+ .await
+ .map_err(|e| {
+ napi::Error::new(
+ napi::Status::GenericFailure,
+ format!("compute_engpicker_verdict join error: {e}"),
+ )
+ })?
+}
+
+fn _compute_engpicker_verdict(
+ results: Vec<EngpickerUrlResult>,
+ similarity_threshold: f64,
+ success_rate_threshold: f64,
+ cdp_failure_threshold: f64,
+) -> napi::Result<EngpickerVerdict> {
+ let url_verdicts: Vec<EngpickerUrlVerdict> = results
+ .iter()
+ .map(|result| {
+ // Get the best chrome-cdp result as gold standard (prefer stealth if both succeeded)
+ let gold_standard = if result.cdp_stealth_success && result.cdp_stealth_markdown.is_some() {
+ result.cdp_stealth_markdown.as_ref()
+ } else if result.cdp_basic_success && result.cdp_basic_markdown.is_some() {
+ result.cdp_basic_markdown.as_ref()
+ } else {
+ None
+ };
+
+ // Get the best tlsclient result (prefer stealth if both succeeded)
+ let tls_result = if result.tls_stealth_success && result.tls_stealth_markdown.is_some() {
+ result.tls_stealth_markdown.as_ref()
+ } else if result.tls_basic_success && result.tls_basic_markdown.is_some() {
+ result.tls_basic_markdown.as_ref()
+ } else {
+ None
+ };
+
+ // If chrome-cdp failed, we can't evaluate this URL
+ let gold_standard = match gold_standard {
+ Some(gs) if !gs.is_empty() => gs,
+ _ => {
+ return EngpickerUrlVerdict {
+ url: result.url.clone(),
+ tls_client_sufficient: false,
+ cdp_failed: true,
+ similarity: None,
+ reason: "chrome-cdp failed".to_string(),
+ };
+ }
+ };
+
+ // If tlsclient failed entirely, it's definitely not enough
+ let tls_result = match tls_result {
+ Some(tls) if !tls.is_empty() => tls,
+ _ => {
+ return EngpickerUrlVerdict {
+ url: result.url.clone(),
+ tls_client_sufficient: false,
+ cdp_failed: false,
+ similarity: None,
+ reason: "tlsclient failed".to_string(),
+ };
+ }
+ };
+
+ // Calculate Levenshtein distance and normalize to similarity score
+ let distance = levenshtein(gold_standard, tls_result);
+ let max_length = gold_standard.len().max(tls_result.len());
+ let similarity = if max_length > 0 {
+ 1.0 - (distance as f64 / max_length as f64)
+ } else {
+ 1.0
+ };
+
+ let tls_client_sufficient = similarity >= similarity_threshold;
+
+ let reason = if tls_client_sufficient {
+ format!("{:.1}% similar - tlsclient captures full content", similarity * 100.0)
+ } else {
+ format!("{:.1}% similar - JS rendering likely required", similarity * 100.0)
+ };
+
+ EngpickerUrlVerdict {
+ url: result.url.clone(),
+ tls_client_sufficient,
+ cdp_failed: false,
+ similarity: Some(similarity),
+ reason,
+ }
+ })
+ .collect();
+
+ let total_urls = url_verdicts.len() as u32;
+ let cdp_failed_count = url_verdicts.iter().filter(|v| v.cdp_failed).count() as u32;
+ let tls_client_ok_count = url_verdicts.iter().filter(|v| v.tls_client_sufficient).count() as u32;
+ let chrome_cdp_required_count = url_verdicts.iter().filter(|v| !v.tls_client_sufficient && !v.cdp_failed).count() as u32;
+
+ // Determine final verdict
+ let verdict = if total_urls == 0 {
+ EngpickerFinalVerdict::Uncertain
+ } else {
+ let cdp_failure_rate = cdp_failed_count as f64 / total_urls as f64;
+
+ // If too many CDP failures, we can't make a confident verdict
+ if cdp_failure_rate > cdp_failure_threshold {
+ EngpickerFinalVerdict::Uncertain
+ } else {
+ // Calculate success rate among URLs where we could actually compare
+ let comparable_urls = total_urls - cdp_failed_count;
+ if comparable_urls == 0 {
+ EngpickerFinalVerdict::Uncertain
+ } else {
+ let tls_ok_rate = tls_client_ok_count as f64 / comparable_urls as f64;
+ if tls_ok_rate >= success_rate_threshold {
+ EngpickerFinalVerdict::TlsClientOk
+ } else {
+ EngpickerFinalVerdict::ChromeCdpRequired
+ }
+ }
+ }
+ };
+
+ Ok(EngpickerVerdict {
+ url_verdicts,
+ tls_client_ok_count,
+ chrome_cdp_required_count,
+ cdp_failed_count,
+ total_urls,
+ verdict,
+ })
+}
+
diff --git a/apps/api/native/src/lib.rs b/apps/api/native/src/lib.rs
index f5f5c48e7..731c4f2b4 100644
--- a/apps/api/native/src/lib.rs
+++ b/apps/api/native/src/lib.rs
@@ -1,6 +1,7 @@
#![deny(clippy::all)]
pub use crate::crawler::*;
+pub use crate::engpicker::*;
pub use crate::html::*;
pub use crate::pdf::*;
pub use crate::utils::*;
@@ -9,6 +10,7 @@ pub use crate::document::{DocumentConverter, DocumentType};
mod crawler;
mod document;
+mod engpicker;
mod html;
mod pdf;
mod utils;
diff --git a/apps/api/src/lib/engpicker.ts b/apps/api/src/lib/engpicker.ts
new file mode 100644
index 000000000..b517a1899
--- /dev/null
+++ b/apps/api/src/lib/engpicker.ts
@@ -0,0 +1,297 @@
+import { generateObject } from "ai";
+import { z } from "zod";
+import { scrapeOptions } from "../controllers/v2/types";
+import { scrapeURL } from "../scraper/scrapeURL";
+import type { Engine } from "../scraper/scrapeURL/engines";
+import { index_supabase_service } from "../services";
+import { CostTracking } from "./cost-tracking";
+import { getModel } from "./generic-ai";
+import { logger as _logger } from "./logger";
+import type { Logger } from "winston";
+import { computeEngpickerVerdict, type EngpickerUrlResult } from "../../native";
+
+type EngpickerJob = {
+ id: number;
+ domain_hash: string;
+ domain_level: number;
+ picked_up_at: string | null;
+ done: boolean;
+ created_at: string;
+};
+
+async function evaluateURL(
+ id: string,
+ url: string,
+ engine: Engine,
+ stealth: boolean,
+ logger: Logger,
+): Promise<{
+ engine: Engine;
+ stealth: boolean;
+ markdown: string | null;
+ result: boolean;
+}> {
+ const scrapeResult = await scrapeURL(
+ id,
+ url,
+ scrapeOptions.parse({
+ proxy: stealth ? "stealth" : "basic",
+ maxAge: 0,
+ storeInCache: false,
+ }),
+ {
+ forceEngine: engine,
+ teamId: "engpicker",
+ },
+ new CostTracking(),
+ );
+
+ if (!scrapeResult.success) {
+ logger.warn("Scrape failed", { scrapeResult });
+ return {
+ engine,
+ stealth,
+ markdown: null,
+ result: false,
+ };
+ }
+
+ const markdown = scrapeResult.document.markdown ?? "";
+
+ logger.info("Scrape completed, waiting for AI evaluation");
+
+ // Use GPT-4o-mini to evaluate if the scrape was actually successful
+ const evaluationResult = await generateObject({
+ model: getModel("gpt-4o-mini", "openai"),
+ schema: z.object({
+ is_successful: z.boolean(),
+ }),
+ messages: [
+ {
+ role: "system",
+ content: `You are a web scraping quality evaluator. Your job is to determine if a web scrape was successful based on the returned markdown content.
+
+A scrape should be considered UNSUCCESSFUL if the content indicates any of the following:
+- Antibot/captcha challenges (e.g., Cloudflare, reCAPTCHA, hCaptcha, bot detection messages)
+- Region/geo blocks (e.g., "not available in your region", "access denied from your location")
+- HTTP error pages (4xx or 5xx errors like "404 Not Found", "403 Forbidden", "500 Internal Server Error")
+- Access denied or authentication required pages
+- Rate limiting messages
+- Empty or near-empty content that suggests the page didn't load properly
+
+A scrape should be considered SUCCESSFUL if:
+- The content appears to be the actual page content with meaningful text
+- The page loaded properly with real content visible
+
+A scrape may still be successful if:
+- Cookie consent walls block the actual content
+- Paywalls block the actual content`,
+ },
+ {
+ role: "user",
+ content: `Evaluate if this scraped markdown content represents a successful scrape:\n\n${markdown.slice(0, 4000)}`,
+ },
+ ],
+ });
+
+ const isSuccess = evaluationResult.object.is_successful;
+
+ logger.info("AI evaluation completed", { isSuccess });
+
+ return {
+ engine,
+ stealth,
+ markdown,
+ result: isSuccess,
+ };
+}
+
+export async function processEngpickerJob() {
+ let logger = _logger.child({
+ module: "engpicker",
+ method: "processEngpickerJob",
+ });
+
+ const { data: jobData, error: getJobError } = await index_supabase_service
+ .from("engpicker_queue")
+ .update({
+ picked_up_at: new Date().toISOString(),
+ })
+ .is("picked_up_at", null)
+ .order("created_at", { ascending: false })
+ .limit(1)
+ .select("*");
+
+ if (getJobError) {
+ logger.error("Error picking up engpicker job", { getJobError });
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ return;
+ }
+
+ const job: EngpickerJob | undefined = jobData?.[0] ?? undefined;
+ if (!job) {
+ logger.debug("No engpicker job to pick up");
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ return;
+ }
+
+ logger = logger.child({
+ jobId: job.id,
+ });
+
+ logger.info("Picked up engpicker job");
+
+ // main shit here
+
+ const { data: indexRows, error: indexRowsError } =
+ await index_supabase_service
+ .rpc("query_index_at_domain_split_level_omce", {
+ i_level: job.domain_level,
+ i_domain_hash: job.domain_hash,
+ i_newer_than: new Date(
+ new Date(job.created_at).valueOf() - 1000 * 60 * 60 * 24,
+ ).toISOString(),
+ })
+ .limit(100);
+
+ if (indexRowsError || !indexRows) {
+ logger.error("Error querying index rows", { indexRowsError });
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ return;
+ }
+
+ const randomizedURLs: string[] = [];
+ for (let i = 0; i < 10; i++) {
+ const elem = indexRows.splice(
+ Math.floor(Math.random() * indexRows.length),
+ 1,
+ );
+ if (elem && elem.length > 0) {
+ randomizedURLs.push(elem[0].url);
+ }
+ }
+
+ logger.info("Picked randomized URLs", { randomizedURLs });
+
+ const results = await Promise.all(
+ randomizedURLs.map(async url => ({
+ url,
+ results: await Promise.all([
+ evaluateURL(
+ job.id + "-cdp-basic",
+ url,
+ "fire-engine;chrome-cdp",
+ false,
+ logger.child({
+ method: "evaluateURL",
+ engine: "fire-engine;chrome-cdp",
+ stealth: false,
+ }),
+ ),
+ evaluateURL(
+ job.id + "-cdp-stealth",
+ url,
+ "fire-engine;chrome-cdp;stealth",
+ true,
+ logger.child({
+ method: "evaluateURL",
+ engine: "fire-engine;chrome-cdp;stealth",
+ stealth: true,
+ }),
+ ),
+ evaluateURL(
+ job.id + "-tlsclient-basic",
+ url,
+ "fire-engine;tlsclient",
+ false,
+ logger.child({
+ method: "evaluateURL",
+ engine: "fire-engine;tlsclient",
+ stealth: false,
+ }),
+ ),
+ evaluateURL(
+ job.id + "-tlsclient-stealth",
+ url,
+ "fire-engine;tlsclient;stealth",
+ true,
+ logger.child({
+ method: "evaluateURL",
+ engine: "fire-engine;tlsclient;stealth",
+ stealth: true,
+ }),
+ ),
+ ]),
+ })),
+ );
+
+ // Transform results into format for native Levenshtein comparison
+ const nativeInput: EngpickerUrlResult[] = results.map(result => {
+ const cdpBasic = result.results.find(
+ r => r.engine === "fire-engine;chrome-cdp",
+ );
+ const cdpStealth = result.results.find(
+ r => r.engine === "fire-engine;chrome-cdp;stealth",
+ );
+ const tlsBasic = result.results.find(
+ r => r.engine === "fire-engine;tlsclient",
+ );
+ const tlsStealth = result.results.find(
+ r => r.engine === "fire-engine;tlsclient;stealth",
+ );
+
+ return {
+ url: result.url,
+ cdpBasicMarkdown: cdpBasic?.markdown ?? undefined,
+ cdpBasicSuccess: cdpBasic?.result ?? false,
+ cdpStealthMarkdown: cdpStealth?.markdown ?? undefined,
+ cdpStealthSuccess: cdpStealth?.result ?? false,
+ tlsBasicMarkdown: tlsBasic?.markdown ?? undefined,
+ tlsBasicSuccess: tlsBasic?.result ?? false,
+ tlsStealthMarkdown: tlsStealth?.markdown ?? undefined,
+ tlsStealthSuccess: tlsStealth?.result ?? false,
+ };
+ });
+
+ // Use native Rust implementation for fast Levenshtein comparison
+ const SIMILARITY_THRESHOLD = 0.85; // 85% similarity means tlsclient is good enough
+ const SUCCESS_RATE_THRESHOLD = 0.7; // 70% of comparable URLs must pass for tlsclient to be OK
+ const CDP_FAILURE_THRESHOLD = 0.5; // If more than 50% of CDP scrapes failed, verdict is uncertain
+ const verdictResult = await computeEngpickerVerdict(
+ nativeInput,
+ SIMILARITY_THRESHOLD,
+ SUCCESS_RATE_THRESHOLD,
+ CDP_FAILURE_THRESHOLD,
+ );
+
+ // This is the verdict - "TlsClientOk", "ChromeCdpRequired", or "Uncertain"
+ const verdict = verdictResult.verdict;
+
+ const { error: insertVerdictError } = await index_supabase_service
+ .from("engpicker_verdicts")
+ .insert({
+ domain_hash: job.domain_hash,
+ verdict,
+ });
+
+ if (insertVerdictError) {
+ logger.error("Error inserting engpicker verdict", { insertVerdictError });
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ return;
+ }
+
+ const { error: updateJobError } = await index_supabase_service
+ .from("engpicker_queue")
+ .update({
+ done: true,
+ })
+ .eq("id", job.id);
+
+ if (updateJobError) {
+ logger.error("Error updating engpicker job", { updateJobError });
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ return;
+ }
+
+ logger.info("Engpicker job completed", { verdict });
+}
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 514668b02..ab58050eb 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -40,6 +40,7 @@ import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
import { crawlGroup } from "../worker/nuq";
import { getACUCTeam } from "../../controllers/auth";
import { supabase_service } from "../supabase";
+import { processEngpickerJob } from "../../lib/engpicker";
const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
const workerStalledCheckInterval =
@@ -773,6 +774,17 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;
5 * 60 * 1000,
);
+ const engpickerPromise = (async () => {
+ while (!isShuttingDown) {
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ try {
+ await processEngpickerJob();
+ } catch (e) {
+ logger.error("Error processing engpicker job", { error: e });
+ }
+ }
+ })();
+
// Search indexing is now handled by separate search service
// The search service has its own worker that processes the queue
// This worker no longer needs to process search index jobs
@@ -795,7 +807,11 @@ const DOMAIN_FREQUENCY_INTERVAL = 10000;
}
// Wait for all workers to complete (which should only happen on shutdown)
- await Promise.all([billingWorkerPromise, precrawlWorkerPromise]);
+ await Promise.all([
+ billingWorkerPromise,
+ precrawlWorkerPromise,
+ engpickerPromise,
+ ]);
clearInterval(indexInserterInterval);
clearInterval(webhookInserterInterval);