The headless browser for AI agents and web scraping
Merge upstream/main and preserve worker readiness options
vvinayakkk215ef5136c8b
diff --cc crates/obscura-cli/src/worker.rsindex 235b3ee6,6c2ef608..b60d3c0a--- a/crates/obscura-cli/src/worker.rs+++ b/crates/obscura-cli/src/worker.rs@@@ -125,12 -96,26 +125,26 @@@ async fn main()};let resp = match cmd {- WorkerCommand::Navigate { url } => {- match page.navigate(&url).await {+ WorkerCommand::Navigate { url, wait_until } => {+ match page.navigate_with_wait(&url, wait_until.into()).await {- Ok(()) => WorkerResponse::success(serde_json::json!({- "title": page.title,- "url": page.url_string(),- })),+ Ok(()) => {+ // Navigation clears the event list and records the main+ // Document response before scripts and child frames.+ // Its status is the final response after HTTP redirects.+ // Zero denotes a blocked request, not an HTTP response.+ let status = page.network_events.iter()+ .find(|event| event.resource_type == "Document")+ .filter(|event| {+ event.status != 0+ && (event.url.starts_with("http://") || event.url.starts_with("https://"))+ })+ .map(|event| event.status);+ WorkerResponse::success(serde_json::json!({+ "title": page.title,+ "url": page.url_string(),+ "status": status,+ }))+ },Err(e) => WorkerResponse::error(e.to_string()),}}diff --cc crates/obscura-cli/tests/worker_navigation_wait.rsindex 57a915cd,00000000..de22ab2amode 100644,000000..100644--- a/crates/obscura-cli/tests/worker_navigation_wait.rs+++ b/crates/obscura-cli/tests/worker_navigation_wait.rs@@@ -1,251 -1,0 +1,273 @@@+use std::io::{BufRead, BufReader, Read, Write};+use std::net::{TcpListener, TcpStream};+use std::process::{Child, ChildStdin, Command, Stdio};+use std::sync::{+ atomic::{AtomicBool, Ordering},+ mpsc, Arc,+};+use std::thread::{self, JoinHandle};+use std::time::Duration;++use serde_json::{json, Value};++struct Server {+ address: String,+ stop: Arc<AtomicBool>,+ task: Option<JoinHandle<()>>,+}++impl Server {+ fn new() -> Self {+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();+ listener.set_nonblocking(true).unwrap();+ let address = listener.local_addr().unwrap().to_string();+ let stop = Arc::new(AtomicBool::new(false));+ let stopped = stop.clone();+ let task = thread::spawn(move || {+ while !stopped.load(Ordering::Relaxed) {+ match listener.accept() {+ Ok((stream, _)) => serve(stream),+ Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {+ thread::sleep(Duration::from_millis(2));+ }+ Err(error) => panic!("accept fixture: {error}"),+ }+ }+ });+ Self {+ address,+ stop,+ task: Some(task),+ }+ }++ fn url(&self, path: &str) -> String {+ format!("http://{}{path}", self.address)+ }+}++impl Drop for Server {+ fn drop(&mut self) {+ self.stop.store(true, Ordering::Relaxed);+ if let Some(task) = self.task.take() {+ task.join().unwrap();+ }+ }+}++fn serve(mut stream: TcpStream) {+ stream.set_nonblocking(false).unwrap();+ stream+ .set_read_timeout(Some(Duration::from_secs(2)))+ .unwrap();+ stream+ .set_write_timeout(Some(Duration::from_secs(2)))+ .unwrap();+ let mut request = Vec::new();+ let mut buffer = [0; 1024];+ while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") {+ let count = match stream.read(&mut buffer) {+ Ok(count) => count,+ Err(_) => return,+ };+ if count == 0 {+ return;+ }+ request.extend_from_slice(&buffer[..count]);+ assert!(request.len() < 8192, "fixture request too large");+ }+ let request = String::from_utf8_lossy(&request);+ let path = request+ .lines()+ .next()+ .unwrap()+ .split_whitespace()+ .nth(1)+ .unwrap();+ if path == "/disconnect" {+ return;+ }- let (status, headers, body) = (200, "", "<title>Ready</title><script>globalThis.inlineRan = true; setTimeout(() => { globalThis.laterRan = true; }, 350);</script><p>Body</p>");++ let (status, headers) = match path {++ "/redirect" => (302, "Location: /missing\r\n"),++ "/missing" => (404, ""),++ _ => (200, ""),++ };++ let body = "<title>Ready</title><script>globalThis.inlineRan = true; setTimeout(() => { globalThis.laterRan = true; }, 350);</script><p>Body</p>";+ let response = format!(+ "HTTP/1.1 {status} Fixture\r\nContent-Type: text/html\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()+ );+ // A navigation cancellation may close the connection before this write.+ let _ = stream.write_all(response.as_bytes());+}++struct Worker {+ child: Child,+ input: ChildStdin,+ responses: mpsc::Receiver<Value>,+ reader: Option<JoinHandle<()>>,+}++impl Worker {+ fn new() -> Self {+ Self::with_stealth(false)+ }++ fn with_stealth(stealth: bool) -> Self {+ let mut child = Command::new(env!("CARGO_BIN_EXE_obscura-worker"))+ .env("OBSCURA_ALLOW_PRIVATE_NETWORK", "1")+ .env("OBSCURA_STEALTH", if stealth { "1" } else { "0" })+ .env("OBSCURA_OBEY_ROBOTS", "0")+ .env("OBSCURA_NAV_TIMEOUT_MS", "5000")+ .env_remove("OBSCURA_PROXY")+ .stdin(Stdio::piped())+ .stdout(Stdio::piped())+ .stderr(Stdio::inherit())+ .spawn()+ .unwrap();+ let input = child.stdin.take().unwrap();+ let output = child.stdout.take().unwrap();+ let (sender, responses) = mpsc::channel();+ let reader = thread::spawn(move || {+ for line in BufReader::new(output).lines() {+ let response = serde_json::from_str(&line.unwrap()).unwrap();+ if sender.send(response).is_err() {+ break;+ }+ }+ });+ Self {+ child,+ input,+ responses,+ reader: Some(reader),+ }+ }++ fn send(&mut self, command: Value) -> Value {+ writeln!(self.input, "{command}").unwrap();+ self.input.flush().unwrap();+ self.responses+ .recv_timeout(Duration::from_secs(10))+ .expect("worker response before deadline")+ }++ fn navigate(&mut self, url: &str) -> Value {+ let response = self.send(json!({ "cmd": "navigate", "url": url }));+ assert_eq!(response["ok"], true, "{response}");+ response["result"].clone()+ }+}++impl Drop for Worker {+ fn drop(&mut self) {+ let _ = self.child.kill();+ let _ = self.child.wait();+ if let Some(reader) = self.reader.take() {+ reader.join().unwrap();+ }+ }+}++fn check_modes(stealth: bool) {+ let server = Server::new();+ let mut worker = Worker::with_stealth(stealth);+ for mode in ["load", "domcontentloaded", "networkidle0", "networkidle2"] {+ let response =+ worker.send(json!({"cmd": "navigate", "url": server.url("/"), "waitUntil": mode}));+ assert_eq!(response["ok"], true, "{mode}: {response}");+ assert_eq!(response["result"]["title"], "Ready");++ assert_eq!(response["result"]["status"], 200);+ let inline = worker.send(json!({"cmd": "evaluate", "expression": "globalThis.inlineRan"}));+ assert_eq!(inline["result"], true, "parser script must run with {mode}");+ if mode.starts_with("networkidle") {+ let later = worker+ .send(json!({"cmd": "evaluate", "expression": "globalThis.laterRan === true"}));+ assert_eq!(+ later["result"], true,+ "{mode} must drive the event loop during its idle window"+ );+ }++ let error_page =++ worker.send(json!({"cmd":"navigate","url":server.url("/redirect"),"waitUntil":mode}));++ assert_eq!(++ error_page["ok"], true,++ "{mode}: HTTP error remains a response"++ );++ assert_eq!(++ error_page["result"]["status"], 404,++ "{mode}: final document status after redirect"++ );++ assert_eq!(error_page["result"]["url"], server.url("/missing"));+ }+ assert_eq!(worker.navigate(&server.url("/"))["title"], "Ready");+}++#[test]+fn every_readiness_mode_runs_parser_scripts_and_idle_modes_drive_timers() {+ check_modes(false);+}++#[cfg(feature = "stealth")]+#[test]+fn readiness_modes_work_with_the_stealth_transport() {+ check_modes(true);+}++#[test]+fn omitted_mode_keeps_load_default_after_a_request_with_an_explicit_mode() {+ let server = Server::new();+ let mut worker = Worker::new();+ worker.send(json!({"cmd": "navigate", "url": server.url("/"), "waitUntil": "networkidle0"}));+ worker.navigate(&server.url("/"));+ let later =+ worker.send(json!({"cmd": "evaluate", "expression": "globalThis.laterRan === true"}));+ assert_eq!(+ later["result"], false,+ "network-idle selection must not leak to the next request"+ );+}++#[test]+fn invalid_modes_are_errors_and_do_not_navigate_or_poison_the_worker() {+ let mut worker = Worker::new();+ worker.navigate("data:text/html,<title>Original</title>");+ for mode in [+ json!("domContentLoaded"),+ json!("networkIdle"),+ json!(""),+ json!(null),+ json!(1),+ json!(true),+ ] {+ let response = worker.send(json!({"cmd": "navigate", "url": "data:text/html,<title>Changed</title>", "waitUntil": mode}));+ assert_eq!(response["ok"], false, "{mode}: {response}");+ assert!(response["error"]+ .as_str()+ .unwrap()+ .starts_with("Invalid command:"));+ assert_eq!(worker.send(json!({"cmd": "title"}))["result"], "Original");+ }+ assert_eq!(+ worker.navigate("data:text/html,<title>Recovered</title>")["title"],+ "Recovered"+ );+}++#[test]+fn readiness_modes_preserve_non_http_navigation() {+ let mut worker = Worker::new();+ for mode in ["load", "domcontentloaded", "networkidle0", "networkidle2"] {+ for url in [+ "about:blank",+ "data:text/html,<title>Data</title><p>Local</p>",+ ] {+ let response = worker.send(json!({"cmd": "navigate", "url": url, "waitUntil": mode}));+ assert_eq!(response["ok"], true, "{mode}: {response}");++ assert_eq!(++ response["result"]["status"],++ Value::Null,++ "{mode}: non-HTTP navigation has no status"++ );+ }+ }+}diff --cc docs/CLI-reference.mdindex d396cf3b,49285755..30edbea3--- a/docs/CLI-reference.md+++ b/docs/CLI-reference.md@@@ -128,24 -128,30 +128,52 @@@ Default transport is stdio. See [Use thRender-enabled builds add `browser_screenshot` and `browser_pdf` to the MCPtool list. Streaming screencasts are available through CDP rather than MCP.+ ## Worker navigation responses++ `obscura-worker` accepts newline-delimited JSON commands on stdin and emits one+ JSON response per command on stdout. For example:++ ```json+ {"cmd":"navigate","url":"https://example.com/"}+ ```++ A completed navigation includes the final document's HTTP status alongside the+ existing title and URL:++ ```json+ {"ok":true,"result":{"title":"Example Domain","url":"https://example.com/","status":200}}+ ```++ `status` is the main document's final HTTP response code after redirects, not a+ script, image, or child frame's response. HTTP errors such as 404 and 503 still+ return `ok: true`: the server responded and the error document can be inspected+ with `dump_html`, `dump_text`, or `evaluate`. Navigation without an HTTP response,+ such as `about:blank`, a `data:` URL, or a tracker blocked before the request,+ returns `status: null`. A transport or+ navigation failure retains the existing `{"ok":false,"error":"..."}` envelope+ without a `result` object.++ The status field is additive. Navigation timing and the other worker commands+ are unchanged.+++## Worker navigation readiness++`obscura-worker` accepts newline-delimited JSON commands on stdin and writes+one response per command to stdout. Select a readiness mode per navigation:++```json+{"cmd":"navigate","url":"https://example.com","waitUntil":"domcontentloaded"}+```++`waitUntil` accepts `load`, `domcontentloaded`, `networkidle0`, or `networkidle2`.+Omitting it preserves the existing `load` default. The option applies only to+that request. Invalid values return an `ok: false` response without navigating;+the worker remains available for subsequent commands.++These modes use the existing `Page::navigate_with_wait` behavior. Parser scripts+run in every mode. Network-idle modes additionally drive the event loop until+active requests remain at or below zero or two, respectively, for 500 ms, with+the engine's existing five-second idle deadline. `domcontentloaded` uses the+engine's DOM readiness boundary; it does not change script execution or resource- warmup semantics. Successful responses retain their existing `title` and `url`++warmup semantics. Successful responses retain their existing `title`, `url`, and `status`+fields.