Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
feat(fetch): enrich JSON dump metadata
ldm0461f9bcac4d9
diff --git a/moli-core/src/page/page_state_cache.rs b/moli-core/src/page/page_state_cache.rsindex c01cedc60..7b182ba34 100644--- a/moli-core/src/page/page_state_cache.rs+++ b/moli-core/src/page/page_state_cache.rs@@ -50,6 +50,10 @@ impl PageStateCache {self.state().navigation_redirect_count}+ pub(super) fn navigation_redirect_chain(&self) -> &[crate::page::NavigationRedirect] {+ &self.state().navigation_redirect_chain+ }+pub(super) fn headers(&self) -> &[(String, String)] {&self.state().headers}diff --git a/moli-core/src/page/renderer_command_support.rs b/moli-core/src/page/renderer_command_support.rsindex 8f6a237d5..15824cd36 100644--- a/moli-core/src/page/renderer_command_support.rs+++ b/moli-core/src/page/renderer_command_support.rs@@ -166,6 +166,10 @@ impl Page {self.page_state.navigation_redirect_count()}+ pub fn navigation_redirect_chain(&self) -> &[super::NavigationRedirect] {+ self.page_state.navigation_redirect_chain()+ }+pub fn final_url(&self) -> &Url {self.page_state.final_url()}diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rsindex 6981bfdc2..55ee58dc4 100644--- a/moli-core/src/runtime/mod.rs+++ b/moli-core/src/runtime/mod.rs@@ -195,6 +195,10 @@ impl RawDocument {&self.response.headers}+ pub fn navigation_redirect_chain(&self) -> &[moli_fetch::RedirectInfo] {+ &self.response.redirect_chain+ }+pub fn body_bytes(&self) -> &[u8] {self.response.body_bytes()}@@ -1200,6 +1204,12 @@ impl Browser {let response_headers = response.headers.clone();let redirected = response.redirected;let redirect_count = response.redirect_chain.len();+ let navigation_redirect_chain = response+ .redirect_chain+ .iter()+ .cloned()+ .map(Into::into)+ .collect();let raw_body =external_raw_document_body_from_streaming_response_with_page_creation_progress(response,@@ -1233,6 +1243,7 @@ impl Browser {None,redirected,redirect_count,+ navigation_redirect_chain,response_status,response_headers,&page_loader,diff --git a/moli-core/src/runtime/navigation_engine.rs b/moli-core/src/runtime/navigation_engine.rsindex 50c0197d6..6bb9cfd68 100644--- a/moli-core/src/runtime/navigation_engine.rs+++ b/moli-core/src/runtime/navigation_engine.rs@@ -1804,6 +1804,7 @@ impl NavigationEngine {navigation_initiator_url,redirected,redirect_count,+ Vec::new(),response_status,response_headers,&loader,diff --git a/moli-core/src/runtime/tests.rs b/moli-core/src/runtime/tests.rsindex aafe1dc1c..9fb92d3c6 100644--- a/moli-core/src/runtime/tests.rs+++ b/moli-core/src/runtime/tests.rs@@ -4476,6 +4476,7 @@ async fn renderer_owner_remove_unknown_page_keeps_never_tracked_state() -> Resulnavigation_initiator_url: None,navigation_redirected: false,navigation_redirect_count: 0,+ navigation_redirect_chain: Vec::new(),final_url: Url::parse("https://example.com/final")?,document_title: String::new(),status: 200,diff --git a/moli-core/tests/fetch_behaviors.rs b/moli-core/tests/fetch_behaviors.rsindex 96af14008..998970653 100644--- a/moli-core/tests/fetch_behaviors.rs+++ b/moli-core/tests/fetch_behaviors.rs@@ -2155,6 +2155,15 @@ async fn carries_and_replaces_cookies_across_multi_hop_redirect_chain() -> Resulassert_eq!(page.status(), 200);assert_eq!(page.final_url().path(), "/redirect-cookie-chain/final");+ assert_eq!(page.navigation_redirect_chain().len(), 2);+ assert_eq!(+ page.navigation_redirect_chain()[0].from_url.path(),+ "/redirect-cookie-chain/start"+ );+ assert_eq!(+ page.navigation_redirect_chain()[1].to_url.path(),+ "/redirect-cookie-chain/final"+ );assert!(page.serialize_html_async().awaitdiff --git a/moli-renderer-v8/src/devtools/ingress/main.rs b/moli-renderer-v8/src/devtools/ingress/main.rsindex e320a4a7a..f029fdbaa 100644--- a/moli-renderer-v8/src/devtools/ingress/main.rs+++ b/moli-renderer-v8/src/devtools/ingress/main.rs@@ -689,6 +689,7 @@ mod tests {navigation_initiator_url: None,navigation_redirected: false,navigation_redirect_count: 0,+ navigation_redirect_chain: Vec::new(),final_url: url,document_title: String::new(),status: 200,diff --git a/moli-renderer-v8/src/runtime/mod.rs b/moli-renderer-v8/src/runtime/mod.rsindex 85d98bd0a..3ef3d2520 100644--- a/moli-renderer-v8/src/runtime/mod.rs+++ b/moli-renderer-v8/src/runtime/mod.rs@@ -653,7 +653,7 @@ impl PageVmPendingPhaseOneNavigation {#[derive(Default)]pub(in crate::runtime) struct PageVmFollowedNavigationMetadata {- pub(super) followed_navigation_response: Option<(Url, u16, Vec<(String, String)>)>,+ pub(super) committed_navigation_response: Option<PageVmNavigationResponse>,pub(super) service_worker_client_navigate:Option<crate::types::ServiceWorkerClientNavigateContinuation>,pub(super) abort_reserved_service_worker_client_id: Option<ServiceWorkerClientId>,@@ -662,8 +662,8 @@ pub(in crate::runtime) struct PageVmFollowedNavigationMetadata {impl PageVmFollowedNavigationMetadata {fn attach_committed_response(&mut self, page_vm: &mut PageVm) {- if let Some((url, status, headers)) = self.followed_navigation_response.take() {- page_vm::attach_navigation_response_to_page_vm(page_vm, url, status, headers);+ if let Some(response) = self.committed_navigation_response.take() {+ page_vm::attach_navigation_response_to_page_vm(page_vm, response);}}diff --git a/moli-renderer-v8/src/runtime/owner.rs b/moli-renderer-v8/src/runtime/owner.rsindex 54531301e..17510c786 100644--- a/moli-renderer-v8/src/runtime/owner.rs+++ b/moli-renderer-v8/src/runtime/owner.rs@@ -187,6 +187,7 @@ pub struct RendererCreateStreamingRawPageRequest {pub navigation_initiator_url: Option<Url>,pub navigation_redirected: bool,pub navigation_redirect_count: usize,+ pub navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,pub response_status: u16,pub response_headers: Vec<(String, String)>,pub loader: ResourceRequestClient,@@ -2093,6 +2094,7 @@ impl RendererOwnerHandle {navigation_initiator_url: Option<Url>,navigation_redirected: bool,navigation_redirect_count: usize,+ navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,response_status: u16,response_headers: Vec<(String, String)>,loader: &ResourceRequestClient,@@ -2123,6 +2125,7 @@ impl RendererOwnerHandle {navigation_initiator_url,navigation_redirected,navigation_redirect_count,+ navigation_redirect_chain,response_status,response_headers,loader: loader.clone(),@@ -6872,6 +6875,7 @@ impl RendererOwnerHandle {navigation_initiator_url,navigation_redirected,navigation_redirect_count,+ navigation_redirect_chain,response_status,response_headers,loader,@@ -7061,6 +7065,14 @@ impl RendererOwnerHandle {} = *result;match outcome {ParseTimePageVmCreationOutcome::PendingPhaseOne(residence) => {+ let committed_navigation_response = Some(PageVmNavigationResponse {+ requested_url: requested_url.clone(),+ redirected: navigation_redirected,+ redirect_count: navigation_redirect_count,+ redirect_chain: navigation_redirect_chain.clone(),+ status: response_status,+ headers: response_headers.clone(),+ });self.continue_pending_phase_one_page_creation(requested_url,navigation_initiator_url,@@ -7070,7 +7082,10 @@ impl RendererOwnerHandle {response_headers,PageVmPendingPhaseOneNavigation::new(residence,- PageVmFollowedNavigationMetadata::default(),+ PageVmFollowedNavigationMetadata {+ committed_navigation_response,+ ..Default::default()+ },),stage,reply_boundary,@@ -7080,7 +7095,21 @@ impl RendererOwnerHandle {).await}- ParseTimePageVmCreationOutcome::TriggeredNavigation { page_vm, stage } => {+ ParseTimePageVmCreationOutcome::TriggeredNavigation {+ mut page_vm,+ stage,+ } => {+ page_vm::attach_navigation_response_to_page_vm(+ &mut page_vm,+ PageVmNavigationResponse {+ requested_url: requested_url.clone(),+ redirected: navigation_redirected,+ redirect_count: navigation_redirect_count,+ redirect_chain: navigation_redirect_chain.clone(),+ status: response_status,+ headers: response_headers.clone(),+ },+ );self.continue_page_creation_with_pending_navigation(requested_url,navigation_initiator_url,@@ -7098,28 +7127,41 @@ impl RendererOwnerHandle {.await}ParseTimePageVmCreationOutcome::ContinuePhaseTwo {- page_vm,+ mut page_vm,page_tasks,stage,started,- } => RenderRuntimeDispatchOutcome::ContinueNextTurn(Box::new(- RenderRuntimeTurn::FinishHtmlCreatePage {- requested_url,- navigation_initiator_url,- navigation_redirected,- navigation_redirect_count,- response_status,- response_headers,- page_vm: Box::new(page_vm),- page_tasks,- stage,- started,- reply_boundary,- lifecycle_decider,- top_level_navigation_dispatch,- navigation_reply_policy,- },- )),+ } => {+ page_vm::attach_navigation_response_to_page_vm(+ &mut page_vm,+ PageVmNavigationResponse {+ requested_url: requested_url.clone(),+ redirected: navigation_redirected,+ redirect_count: navigation_redirect_count,+ redirect_chain: navigation_redirect_chain,+ status: response_status,+ headers: response_headers.clone(),+ },+ );+ RenderRuntimeDispatchOutcome::ContinueNextTurn(Box::new(+ RenderRuntimeTurn::FinishHtmlCreatePage {+ requested_url,+ navigation_initiator_url,+ navigation_redirected,+ navigation_redirect_count,+ response_status,+ response_headers,+ page_vm: Box::new(page_vm),+ page_tasks,+ stage,+ started,+ reply_boundary,+ lifecycle_decider,+ top_level_navigation_dispatch,+ navigation_reply_policy,+ },+ ))+ }}}Err(error) => Err(error).into(),diff --git a/moli-renderer-v8/src/runtime/page.rs b/moli-renderer-v8/src/runtime/page.rsindex 326d9183c..49d3cb94f 100644--- a/moli-renderer-v8/src/runtime/page.rs+++ b/moli-renderer-v8/src/runtime/page.rs@@ -38,6 +38,9 @@ pub(crate) struct PageVmStateCapture {#[derive(Debug, Clone)]pub(crate) struct PageVmNavigationResponse {pub(crate) requested_url: Url,+ pub(crate) redirected: bool,+ pub(crate) redirect_count: usize,+ pub(crate) redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,pub(crate) status: u16,pub(crate) headers: Vec<(String, String)>,}@@ -642,6 +645,7 @@ impl JsRuntime {navigation_initiator_url: Option<Url>,navigation_redirected: bool,navigation_redirect_count: usize,+ navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,response_status: u16,response_headers: Vec<(String, String)>,loader: &ResourceRequestClient,@@ -679,6 +683,7 @@ impl JsRuntime {navigation_initiator_url,navigation_redirected,navigation_redirect_count,+ navigation_redirect_chain,response_status,response_headers,loader,@@ -722,6 +727,7 @@ impl JsRuntime {navigation_initiator_url: Option<Url>,navigation_redirected: bool,navigation_redirect_count: usize,+ navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,response_status: u16,response_headers: Vec<(String, String)>,loader: &ResourceRequestClient,@@ -768,6 +774,7 @@ impl JsRuntime {navigation_initiator_url,navigation_redirected,navigation_redirect_count,+ navigation_redirect_chain,response_status,response_headers,loader,@@ -816,6 +823,7 @@ impl JsRuntime {navigation_initiator_url: Option<Url>,navigation_redirected: bool,navigation_redirect_count: usize,+ navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,response_status: u16,response_headers: Vec<(String, String)>,loader: &ResourceRequestClient,@@ -857,6 +865,7 @@ impl JsRuntime {navigation_initiator_url,navigation_redirected,navigation_redirect_count,+ navigation_redirect_chain,response_status,response_headers,loader,diff --git a/moli-renderer-v8/src/runtime/page_state.rs b/moli-renderer-v8/src/runtime/page_state.rsindex 6ab872fbc..036fdc9d1 100644--- a/moli-renderer-v8/src/runtime/page_state.rs+++ b/moli-renderer-v8/src/runtime/page_state.rs@@ -13,6 +13,7 @@ pub struct RendererPageState {pub navigation_initiator_url: Option<Url>,pub navigation_redirected: bool,pub navigation_redirect_count: usize,+ pub navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,pub final_url: Url,pub document_title: String,pub status: u16,@@ -28,14 +29,18 @@ impl RendererPageState {pub(crate) fn from_vm_state_capture(mut requested_url: Url,navigation_initiator_url: Option<Url>,- navigation_redirected: bool,- navigation_redirect_count: usize,+ mut navigation_redirected: bool,+ mut navigation_redirect_count: usize,mut status: u16,mut headers: Vec<(String, String)>,state_capture: PageVmStateCapture,) -> Arc<Self> {+ let mut navigation_redirect_chain = Vec::new();if let Some(navigation) = state_capture.navigation_response.as_ref() {requested_url = navigation.requested_url.clone();+ navigation_redirected = navigation.redirected;+ navigation_redirect_count = navigation.redirect_count;+ navigation_redirect_chain = navigation.redirect_chain.clone();status = navigation.status;headers = navigation.headers.clone();}@@ -45,6 +50,7 @@ impl RendererPageState {navigation_initiator_url,navigation_redirected,navigation_redirect_count,+ navigation_redirect_chain,final_url: state_capture.final_url,document_title: state_capture.document_title,status,diff --git a/moli-renderer-v8/src/runtime/page_surface.rs b/moli-renderer-v8/src/runtime/page_surface.rsindex 072b91f31..5f8f36c49 100644--- a/moli-renderer-v8/src/runtime/page_surface.rs+++ b/moli-renderer-v8/src/runtime/page_surface.rs@@ -5310,6 +5310,7 @@ impl RendererPageTable {navigation_initiator_url: None,navigation_redirected: false,navigation_redirect_count: 0,+ navigation_redirect_chain: Vec::new(),final_url,document_title,status,diff --git a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rsindex 413063cb2..b3a3abbd8 100644--- a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs+++ b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs@@ -191,7 +191,15 @@ pub(super) async fn bootstrap_committed_followed_location_navigation();let local_executor_clone = local_executor.clone();let loader_for_bootstrap = loader.clone();- let (requested_url, closed_message, bootstrap): (+ let (+ requested_url,+ navigation_redirected,+ navigation_redirect_chain,+ closed_message,+ bootstrap,+ ): (+ _,+ _,_,_,Pin<@@ -214,6 +222,13 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(requested_url,response,} => {+ let navigation_redirected = response.redirected;+ let navigation_redirect_chain = response+ .redirect_chain+ .iter()+ .cloned()+ .map(Into::into)+ .collect();let bootstrap = Box::pin(async move {let started = Instant::now();let result = match boundary {@@ -249,6 +264,8 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(});(requested_url,+ navigation_redirected,+ navigation_redirect_chain,"committed location-navigation bootstrap local task channel closed",bootstrap,)@@ -301,6 +318,8 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(});(requested_url,+ false,+ Vec::new(),"committed local-response navigation bootstrap local task channel closed",bootstrap,)@@ -315,9 +334,14 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(.await?;attach_followed_navigation_response(&mut page_vm_outcome,- requested_url,- response_status,- response_headers,+ PageVmNavigationResponse {+ requested_url,+ redirected: navigation_redirected,+ redirect_count: navigation_redirect_chain.len(),+ redirect_chain: navigation_redirect_chain,+ status: response_status,+ headers: response_headers,+ },);Ok(page_vm_outcome)}@@ -394,36 +418,22 @@ async fn streaming_navigation_result_to_turn_outcome(pub(in crate::runtime) fn attach_navigation_response_to_page_vm(page_vm: &mut PageVm,- requested_url: Url,- response_status: u16,- response_headers: Vec<(String, String)>,+ response: PageVmNavigationResponse,) {- page_vm.navigation_response = Some(PageVmNavigationResponse {- requested_url,- status: response_status,- headers: response_headers,- });+ page_vm.navigation_response = Some(response);}fn attach_followed_navigation_response(page_vm_outcome: &mut PageVmFollowedNavigationBuildOutcome,- requested_url: Url,- response_status: u16,- response_headers: Vec<(String, String)>,+ response: PageVmNavigationResponse,) {match page_vm_outcome {PageVmFollowedNavigationBuildOutcome::ContinuePostParseLifecycle { page_vm, .. }| PageVmFollowedNavigationBuildOutcome::TriggeredNavigation { page_vm, .. } => {- attach_navigation_response_to_page_vm(- page_vm,- requested_url,- response_status,- response_headers,- );+ attach_navigation_response_to_page_vm(page_vm, response);}PageVmFollowedNavigationBuildOutcome::PendingPhaseOne(pending) => {- pending.metadata.followed_navigation_response =- Some((requested_url, response_status, response_headers));+ pending.metadata.committed_navigation_response = Some(response);}PageVmFollowedNavigationBuildOutcome::Download(_) => {}}diff --git a/moli-renderer-v8/src/runtime/tests.rs b/moli-renderer-v8/src/runtime/tests.rsindex 9dc0737eb..a88ec791f 100644--- a/moli-renderer-v8/src/runtime/tests.rs+++ b/moli-renderer-v8/src/runtime/tests.rs@@ -78,6 +78,7 @@ async fn prepare_test_external_raw_document_with_content_type_and_reply_boundaryNone,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), content_type.to_owned())],loader,@@ -2158,6 +2159,7 @@ async fn external_raw_streaming_page_command_builds_phase_one_page() {None,true,1,+ Vec::new(),203,vec![("content-type".to_owned(),@@ -2888,6 +2890,7 @@ async fn external_raw_streaming_empty_document_reaches_dom_content_loaded() {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -3201,6 +3204,7 @@ async fn external_raw_streaming_delegates_post_load_meta_refresh_to_browser() {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -3289,6 +3293,7 @@ async fn external_raw_streaming_defers_dcl_handler_navigation_to_page_reply() {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -3375,6 +3380,7 @@ globalThis.__ordinaryAfterDcl = new Promise(resolve => {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -3467,6 +3473,7 @@ async fn renderer_owned_navigation_survives_page_creation_observer_detach() {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -16115,6 +16122,7 @@ async fn domcontentloaded_page_creation_reply_resumes_owner_to_load_without_exteNone,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -16202,6 +16210,7 @@ async fn document_commit_background_dcl_completion_resumes_owner_to_load() {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -16516,6 +16525,7 @@ async fn owner_loop_completes_post_dcl_async_script_without_wait_command() {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -16904,6 +16914,7 @@ document.addEventListener("DOMContentLoaded", () => {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -17030,6 +17041,7 @@ blocked pageNone,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,@@ -17560,6 +17572,7 @@ async fn create_test_html_page_at_document_commit_with_navigation_dispatch(None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],loader,diff --git a/moli-renderer-v8/src/runtime/tests/open_streaming.rs b/moli-renderer-v8/src/runtime/tests/open_streaming.rsindex 68eb8de0b..6560481d9 100644--- a/moli-renderer-v8/src/runtime/tests/open_streaming.rs+++ b/moli-renderer-v8/src/runtime/tests/open_streaming.rs@@ -183,6 +183,7 @@ impl OpenStreamingPage {None,false,0,+ Vec::new(),200,vec![("content-type".to_owned(), "text/html".to_owned())],&loader,diff --git a/moli-test-support/src/lib.rs b/moli-test-support/src/lib.rsindex 7648ef119..ff4f8f7af 100644--- a/moli-test-support/src/lib.rs+++ b/moli-test-support/src/lib.rs@@ -82,8 +82,7 @@ const COOKIE_REPLACE_RED_HTML: &str ="<!doctype html><html><body><main>replace-cookie=red</main></body></html>";const COOKIE_REPLACE_BLUE_HTML: &str ="<!doctype html><html><body><main>replace-cookie=blue</main></body></html>";-const COOKIE_CHAIN_OK_HTML: &str =- "<!doctype html><html><body><main>cookie-chain=ok</main></body></html>";+const COOKIE_CHAIN_OK_HTML: &str = "<!doctype html><html><head><title>Cookie chain final</title></head><body><main>cookie-chain=ok</main></body></html>";const COOKIE_CHAIN_BROKEN_HTML: &str ="<!doctype html><html><body><main>cookie-chain=broken</main></body></html>";const COOKIE_LOCATION_GATE_MISSING_HTML: &str = "<!doctype html><html><body><main>cookie-location-gate=missing</main><script>document.cookie='ttwid=fixture; Path=/; Max-Age=3600; SameSite=Lax'; if (!location.search.includes('wid=')) location.replace(location.pathname + '?wid=fixture');</script></body></html>";diff --git a/moli-test-support/src/routes_core.rs b/moli-test-support/src/routes_core.rsindex 06420e23d..02264ee16 100644--- a/moli-test-support/src/routes_core.rs+++ b/moli-test-support/src/routes_core.rs@@ -506,6 +506,10 @@ pub(super) async fn location_nav_assign_source_page() -> Html<&'static str> {Html(LOCATION_NAV_ASSIGN_SOURCE_HTML)}+pub(super) async fn location_nav_http_redirect_source_page() -> Html<&'static str> {+ Html("<!doctype html><html><body><script>location.assign('/redirect')</script></body></html>")+}+pub(super) async fn location_nav_href_source_page() -> Html<&'static str> {Html(LOCATION_NAV_HREF_SOURCE_HTML)}diff --git a/moli-test-support/src/server_routes.rs b/moli-test-support/src/server_routes.rsindex bf52c9738..2a0dec382 100644--- a/moli-test-support/src/server_routes.rs+++ b/moli-test-support/src/server_routes.rs@@ -107,6 +107,10 @@ pub(super) fn build_router() -> Router {"/location-nav/assign-source",get(location_nav_assign_source_page),)+ .route(+ "/location-nav/http-redirect-source",+ get(location_nav_http_redirect_source_page),+ ).route("/location-nav/href-source",get(location_nav_href_source_page),diff --git a/moli/src/fetch_dump/dom.rs b/moli/src/fetch_dump/dom.rsindex 0ed704f4e..0a0064d9c 100644--- a/moli/src/fetch_dump/dom.rs+++ b/moli/src/fetch_dump/dom.rs@@ -37,19 +37,42 @@ pub(super) async fn render_json(} else {None};- render_json_payload(page.final_url().as_str(), page.status(), &html, network)+ let title = page.document_title();+ render_json_payload(+ page.final_url().as_str(),+ page.status(),+ Some(&title),+ page.headers(),+ page.navigation_redirect_chain(),+ &html,+ network,+ )}pub(super) fn render_json_payload(final_url: &str,status: u16,+ title: Option<&str>,+ headers: &[(String, String)],+ redirect_chain: &[moli_core::page::NavigationRedirect],html: &str,network: Option<Value>,) -> Result<String> {- // Keep --dump json as a small stable machine interface for scrapling-style callers.+ // Keep --dump json as a stable machine interface for scrapling-style callers.let mut payload = Map::new();payload.insert("final_url".to_owned(), json!(final_url));payload.insert("status".to_owned(), json!(status));+ payload.insert("title".to_owned(), json!(title));+ payload.insert("headers".to_owned(), render_headers(headers));+ payload.insert(+ "redirect_chain".to_owned(),+ Value::Array(+ redirect_chain+ .iter()+ .map(render_navigation_redirect)+ .collect(),+ ),+ );payload.insert("html".to_owned(), json!(html));if let Some(network) = network {payload.insert("network".to_owned(), network);@@ -57,6 +80,24 @@ pub(super) fn render_json_payload(Ok(serde_json::to_string_pretty(&Value::Object(payload))?)}+fn render_headers(headers: &[(String, String)]) -> Value {+ Value::Array(+ headers+ .iter()+ .map(|(name, value)| json!({ "name": name, "value": value }))+ .collect(),+ )+}++fn render_navigation_redirect(redirect: &moli_core::page::NavigationRedirect) -> Value {+ json!({+ "from_url": redirect.from_url.as_str(),+ "to_url": redirect.to_url.as_str(),+ "status": redirect.status,+ "headers": render_headers(&redirect.headers),+ })+}+pub(super) async fn render_html(page: &mut Page,strip: StripOptions,diff --git a/moli/src/fetch_dump/mod.rs b/moli/src/fetch_dump/mod.rsindex 9247adc1d..6e4258696 100644--- a/moli/src/fetch_dump/mod.rs+++ b/moli/src/fetch_dump/mod.rs@@ -72,8 +72,21 @@ pub fn render_raw_document_dump(DumpFormat::Html => Ok(raw.body_bytes().to_vec()),DumpFormat::Json => {let html = String::from_utf8_lossy(raw.body_bytes());- let payload =- dom::render_json_payload(raw.final_url().as_str(), raw.status(), &html, None)?;+ let redirect_chain = raw+ .navigation_redirect_chain()+ .iter()+ .cloned()+ .map(Into::into)+ .collect::<Vec<_>>();+ let payload = dom::render_json_payload(+ raw.final_url().as_str(),+ raw.status(),+ None,+ raw.headers(),+ &redirect_chain,+ &html,+ None,+ )?;Ok(payload.into_bytes())}DumpFormat::Markdowndiff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rsindex 3a6027370..08174815e 100644--- a/moli/tests/fetch_cli.rs+++ b/moli/tests/fetch_cli.rs@@ -216,9 +216,25 @@ fn assert_json_dump_shape(payload: &Value, expected_url: &str, expected_status:let mut keys = object.keys().cloned().collect::<Vec<_>>();keys.sort();- assert_eq!(keys, vec!["final_url", "html", "status"]);+ assert_eq!(+ keys,+ vec![+ "final_url",+ "headers",+ "html",+ "redirect_chain",+ "status",+ "title",+ ]+ );assert_eq!(payload["final_url"], expected_url);assert_eq!(payload["status"], expected_status);+ assert!(+ payload["title"].is_string() || payload["title"].is_null(),+ "payload={payload}"+ );+ assert!(payload["headers"].is_array(), "payload={payload}");+ assert!(payload["redirect_chain"].is_array(), "payload={payload}");assert!(payload["html"].is_string(), "payload={payload}");}@@ -2194,6 +2210,35 @@ fn cli_dump_json_reports_redirected_final_url() -> Result<()> {Ok(())}+#[test]+fn cli_dump_json_reports_redirect_chain_after_script_navigation() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/location-nav/http-redirect-source");+ let redirect_url = server.url("/redirect");+ let final_url = server.url("/static");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "moli fetch failed: stdout={}\nstderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &final_url, 200);+ let redirects = payload["redirect_chain"]+ .as_array()+ .expect("redirect_chain must be an array");+ assert_eq!(redirects.len(), 1, "stdout={stdout}");+ assert_eq!(redirects[0]["from_url"], redirect_url);+ assert_eq!(redirects[0]["to_url"], final_url);+ assert_eq!(redirects[0]["status"], 307);+ Ok(())+}+#[test]fn cli_domstable_waits_for_late_content() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;@@ -2731,6 +2776,61 @@ fn cli_http_redirect_with_location_does_not_consume_redirect_wait() -> Result<()Ok(())}+#[test]+fn cli_dump_json_includes_title_headers_and_redirect_chain() -> Result<()> {+ let runtime = tokio::runtime::Runtime::new()?;+ let server = runtime.block_on(FixtureServer::spawn())?;+ let url = server.url("/redirect-cookie-chain/start");+ let middle_url = server.url("/redirect-cookie-chain/middle");+ let final_url = server.url("/redirect-cookie-chain/final");+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;+ runtime.block_on(server.shutdown());++ assert!(+ output.status.success(),+ "moli fetch failed: stdout={}\nstderr={}",+ String::from_utf8_lossy(&output.stdout),+ String::from_utf8_lossy(&output.stderr)+ );+ let stdout = clean_output(&output.stdout);+ let payload: Value = serde_json::from_str(&stdout)?;+ assert_json_dump_shape(&payload, &final_url, 200);+ assert_eq!(payload["title"], "Cookie chain final");+ assert!(+ payload["headers"].as_array().is_some_and(|headers| {+ headers.iter().any(|header| {+ header["name"]+ .as_str()+ .is_some_and(|name| name.eq_ignore_ascii_case("content-type"))+ })+ }),+ "stdout={stdout}"+ );++ let redirects = payload["redirect_chain"]+ .as_array()+ .expect("redirect_chain must be an array");+ assert_eq!(redirects.len(), 2, "stdout={stdout}");+ assert_eq!(redirects[0]["from_url"], url);+ assert_eq!(redirects[0]["to_url"], middle_url);+ assert_eq!(redirects[0]["status"], 307);+ assert_eq!(redirects[1]["from_url"], middle_url);+ assert_eq!(redirects[1]["to_url"], final_url);+ assert_eq!(redirects[1]["status"], 307);+ let first_hop_set_cookies = redirects[0]["headers"]+ .as_array()+ .expect("redirect headers must be an array")+ .iter()+ .filter(|header| {+ header["name"]+ .as_str()+ .is_some_and(|name| name.eq_ignore_ascii_case("set-cookie"))+ })+ .count();+ assert_eq!(first_hop_set_cookies, 2, "stdout={stdout}");+ Ok(())+}+#[test]fn cli_domcontentloaded_recovers_from_delayed_403_navigation_at_same_stage() -> Result<()> {let runtime = tokio::runtime::Runtime::new()?;diff --git a/skills/moli-webfetch/SKILL.md b/skills/moli-webfetch/SKILL.mdindex cdc55ff64..5896ac2ef 100644--- a/skills/moli-webfetch/SKILL.md+++ b/skills/moli-webfetch/SKILL.md@@ -81,8 +81,9 @@ structure-first; enable layout only when the result needs pixels or pagination.- Use `markdown` for prose, documentation, articles, and direct model reading.- Use `semantic_tree_text` when navigation-heavy markup makes Markdown noisy orwhen roles and accessible names matter.-- Use `json` for automation that needs `final_url`, HTTP `status`, serialized- `html`, or network trace data.+- Use `json` for automation that needs `final_url`, HTTP `status`, `title`,+ duplicate-safe response `headers`, the main-navigation `redirect_chain`,+ serialized `html`, or network trace data.- Use `html` to diagnose DOM serialization or preserve exact markup.- Use `screenshot` for a viewport PNG when appearance is evidence. It requires`--layout`.diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.mdindex 0c7bef55b..719bdcf2e 100644--- a/skills/moli-webfetch/references/fetch-recipes.md+++ b/skills/moli-webfetch/references/fetch-recipes.md@@ -17,7 +17,7 @@| Read page content | `--dump markdown` | Default for research and summarization || Inspect semantic structure | `--dump semantic_tree_text` | Compact roles, labels, text, and backend node IDs || Process semantic structure | `--dump semantic_tree` | Structured accessibility-oriented payload |-| Process stable fields | `--dump json` | Returns `final_url`, `status`, and `html` |+| Process stable fields | `--dump json` | Returns `final_url`, `status`, `title`, `headers`, `redirect_chain`, and `html` || Inspect exact DOM | `--dump html` | Useful when Markdown loses important structure || Diagnose requests | `--dump json --trace-network` | Adds the `network` object || Capture the viewport | `--layout --dump screenshot` | Writes PNG bytes to stdout |@@ -26,6 +26,10 @@Raw non-HTML responses support only `html` and `json`.+JSON `headers` is an ordered list of `{name, value}` records so duplicate+headers are preserved. `redirect_chain` contains every main-navigation HTTP+redirect hop in order.+## ReadinessStart with `--wait-until done`. Change or extend it only when the page exposes