Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

feat(fetch): enrich JSON dump metadata

ldm0461f9bcac4d9
diff --git a/moli-core/src/page/page_state_cache.rs b/moli-core/src/page/page_state_cache.rs
index c01cedc60..7b182ba34 100644
--- a/moli-core/src/page/page_state_cache.rs
+++ b/moli-core/src/page/page_state_cache.rs
@@ -50,6 +50,10 @@ impl PageStateCache {
self.state().navigation_redirect_count
}
+ pub(super) fn navigation_redirect_chain(&self) -> &[crate::page::NavigationRedirect] {
+ &self.state().navigation_redirect_chain
+ }
+
pub(super) fn headers(&self) -> &[(String, String)] {
&self.state().headers
}
diff --git a/moli-core/src/page/renderer_command_support.rs b/moli-core/src/page/renderer_command_support.rs
index 8f6a237d5..15824cd36 100644
--- a/moli-core/src/page/renderer_command_support.rs
+++ b/moli-core/src/page/renderer_command_support.rs
@@ -166,6 +166,10 @@ impl Page {
self.page_state.navigation_redirect_count()
}
+ pub fn navigation_redirect_chain(&self) -> &[super::NavigationRedirect] {
+ self.page_state.navigation_redirect_chain()
+ }
+
pub fn final_url(&self) -> &Url {
self.page_state.final_url()
}
diff --git a/moli-core/src/runtime/mod.rs b/moli-core/src/runtime/mod.rs
index 6981bfdc2..55ee58dc4 100644
--- a/moli-core/src/runtime/mod.rs
+++ b/moli-core/src/runtime/mod.rs
@@ -195,6 +195,10 @@ impl RawDocument {
&self.response.headers
}
+ pub fn navigation_redirect_chain(&self) -> &[moli_fetch::RedirectInfo] {
+ &self.response.redirect_chain
+ }
+
pub fn body_bytes(&self) -> &[u8] {
self.response.body_bytes()
}
@@ -1200,6 +1204,12 @@ impl Browser {
let response_headers = response.headers.clone();
let redirected = response.redirected;
let redirect_count = response.redirect_chain.len();
+ let navigation_redirect_chain = response
+ .redirect_chain
+ .iter()
+ .cloned()
+ .map(Into::into)
+ .collect();
let raw_body =
external_raw_document_body_from_streaming_response_with_page_creation_progress(
response,
@@ -1233,6 +1243,7 @@ impl Browser {
None,
redirected,
redirect_count,
+ navigation_redirect_chain,
response_status,
response_headers,
&page_loader,
diff --git a/moli-core/src/runtime/navigation_engine.rs b/moli-core/src/runtime/navigation_engine.rs
index 50c0197d6..6bb9cfd68 100644
--- a/moli-core/src/runtime/navigation_engine.rs
+++ b/moli-core/src/runtime/navigation_engine.rs
@@ -1804,6 +1804,7 @@ impl NavigationEngine {
navigation_initiator_url,
redirected,
redirect_count,
+ Vec::new(),
response_status,
response_headers,
&loader,
diff --git a/moli-core/src/runtime/tests.rs b/moli-core/src/runtime/tests.rs
index aafe1dc1c..9fb92d3c6 100644
--- a/moli-core/src/runtime/tests.rs
+++ b/moli-core/src/runtime/tests.rs
@@ -4476,6 +4476,7 @@ async fn renderer_owner_remove_unknown_page_keeps_never_tracked_state() -> Resul
navigation_initiator_url: None,
navigation_redirected: false,
navigation_redirect_count: 0,
+ navigation_redirect_chain: Vec::new(),
final_url: Url::parse("https://example.com/final")?,
document_title: String::new(),
status: 200,
diff --git a/moli-core/tests/fetch_behaviors.rs b/moli-core/tests/fetch_behaviors.rs
index 96af14008..998970653 100644
--- a/moli-core/tests/fetch_behaviors.rs
+++ b/moli-core/tests/fetch_behaviors.rs
@@ -2155,6 +2155,15 @@ async fn carries_and_replaces_cookies_across_multi_hop_redirect_chain() -> Resul
assert_eq!(page.status(), 200);
assert_eq!(page.final_url().path(), "/redirect-cookie-chain/final");
+ assert_eq!(page.navigation_redirect_chain().len(), 2);
+ assert_eq!(
+ page.navigation_redirect_chain()[0].from_url.path(),
+ "/redirect-cookie-chain/start"
+ );
+ assert_eq!(
+ page.navigation_redirect_chain()[1].to_url.path(),
+ "/redirect-cookie-chain/final"
+ );
assert!(
page.serialize_html_async()
.await
diff --git a/moli-renderer-v8/src/devtools/ingress/main.rs b/moli-renderer-v8/src/devtools/ingress/main.rs
index e320a4a7a..f029fdbaa 100644
--- a/moli-renderer-v8/src/devtools/ingress/main.rs
+++ b/moli-renderer-v8/src/devtools/ingress/main.rs
@@ -689,6 +689,7 @@ mod tests {
navigation_initiator_url: None,
navigation_redirected: false,
navigation_redirect_count: 0,
+ navigation_redirect_chain: Vec::new(),
final_url: url,
document_title: String::new(),
status: 200,
diff --git a/moli-renderer-v8/src/runtime/mod.rs b/moli-renderer-v8/src/runtime/mod.rs
index 85d98bd0a..3ef3d2520 100644
--- a/moli-renderer-v8/src/runtime/mod.rs
+++ b/moli-renderer-v8/src/runtime/mod.rs
@@ -653,7 +653,7 @@ impl PageVmPendingPhaseOneNavigation {
#[derive(Default)]
pub(in crate::runtime) struct PageVmFollowedNavigationMetadata {
- pub(super) followed_navigation_response: Option<(Url, u16, Vec<(String, String)>)>,
+ pub(super) committed_navigation_response: Option<PageVmNavigationResponse>,
pub(super) service_worker_client_navigate:
Option<crate::types::ServiceWorkerClientNavigateContinuation>,
pub(super) abort_reserved_service_worker_client_id: Option<ServiceWorkerClientId>,
@@ -662,8 +662,8 @@ pub(in crate::runtime) struct PageVmFollowedNavigationMetadata {
impl PageVmFollowedNavigationMetadata {
fn attach_committed_response(&mut self, page_vm: &mut PageVm) {
- if let Some((url, status, headers)) = self.followed_navigation_response.take() {
- page_vm::attach_navigation_response_to_page_vm(page_vm, url, status, headers);
+ if let Some(response) = self.committed_navigation_response.take() {
+ page_vm::attach_navigation_response_to_page_vm(page_vm, response);
}
}
diff --git a/moli-renderer-v8/src/runtime/owner.rs b/moli-renderer-v8/src/runtime/owner.rs
index 54531301e..17510c786 100644
--- a/moli-renderer-v8/src/runtime/owner.rs
+++ b/moli-renderer-v8/src/runtime/owner.rs
@@ -187,6 +187,7 @@ pub struct RendererCreateStreamingRawPageRequest {
pub navigation_initiator_url: Option<Url>,
pub navigation_redirected: bool,
pub navigation_redirect_count: usize,
+ pub navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,
pub response_status: u16,
pub response_headers: Vec<(String, String)>,
pub loader: ResourceRequestClient,
@@ -2093,6 +2094,7 @@ impl RendererOwnerHandle {
navigation_initiator_url: Option<Url>,
navigation_redirected: bool,
navigation_redirect_count: usize,
+ navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,
response_status: u16,
response_headers: Vec<(String, String)>,
loader: &ResourceRequestClient,
@@ -2123,6 +2125,7 @@ impl RendererOwnerHandle {
navigation_initiator_url,
navigation_redirected,
navigation_redirect_count,
+ navigation_redirect_chain,
response_status,
response_headers,
loader: loader.clone(),
@@ -6872,6 +6875,7 @@ impl RendererOwnerHandle {
navigation_initiator_url,
navigation_redirected,
navigation_redirect_count,
+ navigation_redirect_chain,
response_status,
response_headers,
loader,
@@ -7061,6 +7065,14 @@ impl RendererOwnerHandle {
} = *result;
match outcome {
ParseTimePageVmCreationOutcome::PendingPhaseOne(residence) => {
+ let committed_navigation_response = Some(PageVmNavigationResponse {
+ requested_url: requested_url.clone(),
+ redirected: navigation_redirected,
+ redirect_count: navigation_redirect_count,
+ redirect_chain: navigation_redirect_chain.clone(),
+ status: response_status,
+ headers: response_headers.clone(),
+ });
self.continue_pending_phase_one_page_creation(
requested_url,
navigation_initiator_url,
@@ -7070,7 +7082,10 @@ impl RendererOwnerHandle {
response_headers,
PageVmPendingPhaseOneNavigation::new(
residence,
- PageVmFollowedNavigationMetadata::default(),
+ PageVmFollowedNavigationMetadata {
+ committed_navigation_response,
+ ..Default::default()
+ },
),
stage,
reply_boundary,
@@ -7080,7 +7095,21 @@ impl RendererOwnerHandle {
)
.await
}
- ParseTimePageVmCreationOutcome::TriggeredNavigation { page_vm, stage } => {
+ ParseTimePageVmCreationOutcome::TriggeredNavigation {
+ mut page_vm,
+ stage,
+ } => {
+ page_vm::attach_navigation_response_to_page_vm(
+ &mut page_vm,
+ PageVmNavigationResponse {
+ requested_url: requested_url.clone(),
+ redirected: navigation_redirected,
+ redirect_count: navigation_redirect_count,
+ redirect_chain: navigation_redirect_chain.clone(),
+ status: response_status,
+ headers: response_headers.clone(),
+ },
+ );
self.continue_page_creation_with_pending_navigation(
requested_url,
navigation_initiator_url,
@@ -7098,28 +7127,41 @@ impl RendererOwnerHandle {
.await
}
ParseTimePageVmCreationOutcome::ContinuePhaseTwo {
- page_vm,
+ mut page_vm,
page_tasks,
stage,
started,
- } => RenderRuntimeDispatchOutcome::ContinueNextTurn(Box::new(
- RenderRuntimeTurn::FinishHtmlCreatePage {
- requested_url,
- navigation_initiator_url,
- navigation_redirected,
- navigation_redirect_count,
- response_status,
- response_headers,
- page_vm: Box::new(page_vm),
- page_tasks,
- stage,
- started,
- reply_boundary,
- lifecycle_decider,
- top_level_navigation_dispatch,
- navigation_reply_policy,
- },
- )),
+ } => {
+ page_vm::attach_navigation_response_to_page_vm(
+ &mut page_vm,
+ PageVmNavigationResponse {
+ requested_url: requested_url.clone(),
+ redirected: navigation_redirected,
+ redirect_count: navigation_redirect_count,
+ redirect_chain: navigation_redirect_chain,
+ status: response_status,
+ headers: response_headers.clone(),
+ },
+ );
+ RenderRuntimeDispatchOutcome::ContinueNextTurn(Box::new(
+ RenderRuntimeTurn::FinishHtmlCreatePage {
+ requested_url,
+ navigation_initiator_url,
+ navigation_redirected,
+ navigation_redirect_count,
+ response_status,
+ response_headers,
+ page_vm: Box::new(page_vm),
+ page_tasks,
+ stage,
+ started,
+ reply_boundary,
+ lifecycle_decider,
+ top_level_navigation_dispatch,
+ navigation_reply_policy,
+ },
+ ))
+ }
}
}
Err(error) => Err(error).into(),
diff --git a/moli-renderer-v8/src/runtime/page.rs b/moli-renderer-v8/src/runtime/page.rs
index 326d9183c..49d3cb94f 100644
--- a/moli-renderer-v8/src/runtime/page.rs
+++ b/moli-renderer-v8/src/runtime/page.rs
@@ -38,6 +38,9 @@ pub(crate) struct PageVmStateCapture {
#[derive(Debug, Clone)]
pub(crate) struct PageVmNavigationResponse {
pub(crate) requested_url: Url,
+ pub(crate) redirected: bool,
+ pub(crate) redirect_count: usize,
+ pub(crate) redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,
pub(crate) status: u16,
pub(crate) headers: Vec<(String, String)>,
}
@@ -642,6 +645,7 @@ impl JsRuntime {
navigation_initiator_url: Option<Url>,
navigation_redirected: bool,
navigation_redirect_count: usize,
+ navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,
response_status: u16,
response_headers: Vec<(String, String)>,
loader: &ResourceRequestClient,
@@ -679,6 +683,7 @@ impl JsRuntime {
navigation_initiator_url,
navigation_redirected,
navigation_redirect_count,
+ navigation_redirect_chain,
response_status,
response_headers,
loader,
@@ -722,6 +727,7 @@ impl JsRuntime {
navigation_initiator_url: Option<Url>,
navigation_redirected: bool,
navigation_redirect_count: usize,
+ navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,
response_status: u16,
response_headers: Vec<(String, String)>,
loader: &ResourceRequestClient,
@@ -768,6 +774,7 @@ impl JsRuntime {
navigation_initiator_url,
navigation_redirected,
navigation_redirect_count,
+ navigation_redirect_chain,
response_status,
response_headers,
loader,
@@ -816,6 +823,7 @@ impl JsRuntime {
navigation_initiator_url: Option<Url>,
navigation_redirected: bool,
navigation_redirect_count: usize,
+ navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,
response_status: u16,
response_headers: Vec<(String, String)>,
loader: &ResourceRequestClient,
@@ -857,6 +865,7 @@ impl JsRuntime {
navigation_initiator_url,
navigation_redirected,
navigation_redirect_count,
+ navigation_redirect_chain,
response_status,
response_headers,
loader,
diff --git a/moli-renderer-v8/src/runtime/page_state.rs b/moli-renderer-v8/src/runtime/page_state.rs
index 6ab872fbc..036fdc9d1 100644
--- a/moli-renderer-v8/src/runtime/page_state.rs
+++ b/moli-renderer-v8/src/runtime/page_state.rs
@@ -13,6 +13,7 @@ pub struct RendererPageState {
pub navigation_initiator_url: Option<Url>,
pub navigation_redirected: bool,
pub navigation_redirect_count: usize,
+ pub navigation_redirect_chain: Vec<crate::protocol_types::NavigationRedirect>,
pub final_url: Url,
pub document_title: String,
pub status: u16,
@@ -28,14 +29,18 @@ impl RendererPageState {
pub(crate) fn from_vm_state_capture(
mut requested_url: Url,
navigation_initiator_url: Option<Url>,
- navigation_redirected: bool,
- navigation_redirect_count: usize,
+ mut navigation_redirected: bool,
+ mut navigation_redirect_count: usize,
mut status: u16,
mut headers: Vec<(String, String)>,
state_capture: PageVmStateCapture,
) -> Arc<Self> {
+ let mut navigation_redirect_chain = Vec::new();
if let Some(navigation) = state_capture.navigation_response.as_ref() {
requested_url = navigation.requested_url.clone();
+ navigation_redirected = navigation.redirected;
+ navigation_redirect_count = navigation.redirect_count;
+ navigation_redirect_chain = navigation.redirect_chain.clone();
status = navigation.status;
headers = navigation.headers.clone();
}
@@ -45,6 +50,7 @@ impl RendererPageState {
navigation_initiator_url,
navigation_redirected,
navigation_redirect_count,
+ navigation_redirect_chain,
final_url: state_capture.final_url,
document_title: state_capture.document_title,
status,
diff --git a/moli-renderer-v8/src/runtime/page_surface.rs b/moli-renderer-v8/src/runtime/page_surface.rs
index 072b91f31..5f8f36c49 100644
--- a/moli-renderer-v8/src/runtime/page_surface.rs
+++ b/moli-renderer-v8/src/runtime/page_surface.rs
@@ -5310,6 +5310,7 @@ impl RendererPageTable {
navigation_initiator_url: None,
navigation_redirected: false,
navigation_redirect_count: 0,
+ navigation_redirect_chain: Vec::new(),
final_url,
document_title,
status,
diff --git a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
index 413063cb2..b3a3abbd8 100644
--- a/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
+++ b/moli-renderer-v8/src/runtime/page_vm/followed_navigation.rs
@@ -191,7 +191,15 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(
);
let local_executor_clone = local_executor.clone();
let loader_for_bootstrap = loader.clone();
- let (requested_url, closed_message, bootstrap): (
+ let (
+ requested_url,
+ navigation_redirected,
+ navigation_redirect_chain,
+ closed_message,
+ bootstrap,
+ ): (
+ _,
+ _,
_,
_,
Pin<
@@ -214,6 +222,13 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(
requested_url,
response,
} => {
+ let navigation_redirected = response.redirected;
+ let navigation_redirect_chain = response
+ .redirect_chain
+ .iter()
+ .cloned()
+ .map(Into::into)
+ .collect();
let bootstrap = Box::pin(async move {
let started = Instant::now();
let result = match boundary {
@@ -249,6 +264,8 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(
});
(
requested_url,
+ navigation_redirected,
+ navigation_redirect_chain,
"committed location-navigation bootstrap local task channel closed",
bootstrap,
)
@@ -301,6 +318,8 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(
});
(
requested_url,
+ false,
+ Vec::new(),
"committed local-response navigation bootstrap local task channel closed",
bootstrap,
)
@@ -315,9 +334,14 @@ pub(super) async fn bootstrap_committed_followed_location_navigation(
.await?;
attach_followed_navigation_response(
&mut page_vm_outcome,
- requested_url,
- response_status,
- response_headers,
+ PageVmNavigationResponse {
+ requested_url,
+ redirected: navigation_redirected,
+ redirect_count: navigation_redirect_chain.len(),
+ redirect_chain: navigation_redirect_chain,
+ status: response_status,
+ headers: response_headers,
+ },
);
Ok(page_vm_outcome)
}
@@ -394,36 +418,22 @@ async fn streaming_navigation_result_to_turn_outcome(
pub(in crate::runtime) fn attach_navigation_response_to_page_vm(
page_vm: &mut PageVm,
- requested_url: Url,
- response_status: u16,
- response_headers: Vec<(String, String)>,
+ response: PageVmNavigationResponse,
) {
- page_vm.navigation_response = Some(PageVmNavigationResponse {
- requested_url,
- status: response_status,
- headers: response_headers,
- });
+ page_vm.navigation_response = Some(response);
}
fn attach_followed_navigation_response(
page_vm_outcome: &mut PageVmFollowedNavigationBuildOutcome,
- requested_url: Url,
- response_status: u16,
- response_headers: Vec<(String, String)>,
+ response: PageVmNavigationResponse,
) {
match page_vm_outcome {
PageVmFollowedNavigationBuildOutcome::ContinuePostParseLifecycle { page_vm, .. }
| PageVmFollowedNavigationBuildOutcome::TriggeredNavigation { page_vm, .. } => {
- attach_navigation_response_to_page_vm(
- page_vm,
- requested_url,
- response_status,
- response_headers,
- );
+ attach_navigation_response_to_page_vm(page_vm, response);
}
PageVmFollowedNavigationBuildOutcome::PendingPhaseOne(pending) => {
- pending.metadata.followed_navigation_response =
- Some((requested_url, response_status, response_headers));
+ pending.metadata.committed_navigation_response = Some(response);
}
PageVmFollowedNavigationBuildOutcome::Download(_) => {}
}
diff --git a/moli-renderer-v8/src/runtime/tests.rs b/moli-renderer-v8/src/runtime/tests.rs
index 9dc0737eb..a88ec791f 100644
--- a/moli-renderer-v8/src/runtime/tests.rs
+++ b/moli-renderer-v8/src/runtime/tests.rs
@@ -78,6 +78,7 @@ async fn prepare_test_external_raw_document_with_content_type_and_reply_boundary
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), content_type.to_owned())],
loader,
@@ -2158,6 +2159,7 @@ async fn external_raw_streaming_page_command_builds_phase_one_page() {
None,
true,
1,
+ Vec::new(),
203,
vec![(
"content-type".to_owned(),
@@ -2888,6 +2890,7 @@ async fn external_raw_streaming_empty_document_reaches_dom_content_loaded() {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -3201,6 +3204,7 @@ async fn external_raw_streaming_delegates_post_load_meta_refresh_to_browser() {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -3289,6 +3293,7 @@ async fn external_raw_streaming_defers_dcl_handler_navigation_to_page_reply() {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -3375,6 +3380,7 @@ globalThis.__ordinaryAfterDcl = new Promise(resolve => {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -3467,6 +3473,7 @@ async fn renderer_owned_navigation_survives_page_creation_observer_detach() {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -16115,6 +16122,7 @@ async fn domcontentloaded_page_creation_reply_resumes_owner_to_load_without_exte
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -16202,6 +16210,7 @@ async fn document_commit_background_dcl_completion_resumes_owner_to_load() {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -16516,6 +16525,7 @@ async fn owner_loop_completes_post_dcl_async_script_without_wait_command() {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -16904,6 +16914,7 @@ document.addEventListener("DOMContentLoaded", () => {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -17030,6 +17041,7 @@ blocked page
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
@@ -17560,6 +17572,7 @@ async fn create_test_html_page_at_document_commit_with_navigation_dispatch(
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
loader,
diff --git a/moli-renderer-v8/src/runtime/tests/open_streaming.rs b/moli-renderer-v8/src/runtime/tests/open_streaming.rs
index 68eb8de0b..6560481d9 100644
--- a/moli-renderer-v8/src/runtime/tests/open_streaming.rs
+++ b/moli-renderer-v8/src/runtime/tests/open_streaming.rs
@@ -183,6 +183,7 @@ impl OpenStreamingPage {
None,
false,
0,
+ Vec::new(),
200,
vec![("content-type".to_owned(), "text/html".to_owned())],
&loader,
diff --git a/moli-test-support/src/lib.rs b/moli-test-support/src/lib.rs
index 7648ef119..ff4f8f7af 100644
--- a/moli-test-support/src/lib.rs
+++ b/moli-test-support/src/lib.rs
@@ -82,8 +82,7 @@ const COOKIE_REPLACE_RED_HTML: &str =
"<!doctype html><html><body><main>replace-cookie=red</main></body></html>";
const COOKIE_REPLACE_BLUE_HTML: &str =
"<!doctype html><html><body><main>replace-cookie=blue</main></body></html>";
-const COOKIE_CHAIN_OK_HTML: &str =
- "<!doctype html><html><body><main>cookie-chain=ok</main></body></html>";
+const COOKIE_CHAIN_OK_HTML: &str = "<!doctype html><html><head><title>Cookie chain final</title></head><body><main>cookie-chain=ok</main></body></html>";
const COOKIE_CHAIN_BROKEN_HTML: &str =
"<!doctype html><html><body><main>cookie-chain=broken</main></body></html>";
const COOKIE_LOCATION_GATE_MISSING_HTML: &str = "<!doctype html><html><body><main>cookie-location-gate=missing</main><script>document.cookie='ttwid=fixture; Path=/; Max-Age=3600; SameSite=Lax'; if (!location.search.includes('wid=')) location.replace(location.pathname + '?wid=fixture');</script></body></html>";
diff --git a/moli-test-support/src/routes_core.rs b/moli-test-support/src/routes_core.rs
index 06420e23d..02264ee16 100644
--- a/moli-test-support/src/routes_core.rs
+++ b/moli-test-support/src/routes_core.rs
@@ -506,6 +506,10 @@ pub(super) async fn location_nav_assign_source_page() -> Html<&'static str> {
Html(LOCATION_NAV_ASSIGN_SOURCE_HTML)
}
+pub(super) async fn location_nav_http_redirect_source_page() -> Html<&'static str> {
+ Html("<!doctype html><html><body><script>location.assign('/redirect')</script></body></html>")
+}
+
pub(super) async fn location_nav_href_source_page() -> Html<&'static str> {
Html(LOCATION_NAV_HREF_SOURCE_HTML)
}
diff --git a/moli-test-support/src/server_routes.rs b/moli-test-support/src/server_routes.rs
index bf52c9738..2a0dec382 100644
--- a/moli-test-support/src/server_routes.rs
+++ b/moli-test-support/src/server_routes.rs
@@ -107,6 +107,10 @@ pub(super) fn build_router() -> Router {
"/location-nav/assign-source",
get(location_nav_assign_source_page),
)
+ .route(
+ "/location-nav/http-redirect-source",
+ get(location_nav_http_redirect_source_page),
+ )
.route(
"/location-nav/href-source",
get(location_nav_href_source_page),
diff --git a/moli/src/fetch_dump/dom.rs b/moli/src/fetch_dump/dom.rs
index 0ed704f4e..0a0064d9c 100644
--- a/moli/src/fetch_dump/dom.rs
+++ b/moli/src/fetch_dump/dom.rs
@@ -37,19 +37,42 @@ pub(super) async fn render_json(
} else {
None
};
- render_json_payload(page.final_url().as_str(), page.status(), &html, network)
+ let title = page.document_title();
+ render_json_payload(
+ page.final_url().as_str(),
+ page.status(),
+ Some(&title),
+ page.headers(),
+ page.navigation_redirect_chain(),
+ &html,
+ network,
+ )
}
pub(super) fn render_json_payload(
final_url: &str,
status: u16,
+ title: Option<&str>,
+ headers: &[(String, String)],
+ redirect_chain: &[moli_core::page::NavigationRedirect],
html: &str,
network: Option<Value>,
) -> Result<String> {
- // Keep --dump json as a small stable machine interface for scrapling-style callers.
+ // Keep --dump json as a stable machine interface for scrapling-style callers.
let mut payload = Map::new();
payload.insert("final_url".to_owned(), json!(final_url));
payload.insert("status".to_owned(), json!(status));
+ payload.insert("title".to_owned(), json!(title));
+ payload.insert("headers".to_owned(), render_headers(headers));
+ payload.insert(
+ "redirect_chain".to_owned(),
+ Value::Array(
+ redirect_chain
+ .iter()
+ .map(render_navigation_redirect)
+ .collect(),
+ ),
+ );
payload.insert("html".to_owned(), json!(html));
if let Some(network) = network {
payload.insert("network".to_owned(), network);
@@ -57,6 +80,24 @@ pub(super) fn render_json_payload(
Ok(serde_json::to_string_pretty(&Value::Object(payload))?)
}
+fn render_headers(headers: &[(String, String)]) -> Value {
+ Value::Array(
+ headers
+ .iter()
+ .map(|(name, value)| json!({ "name": name, "value": value }))
+ .collect(),
+ )
+}
+
+fn render_navigation_redirect(redirect: &moli_core::page::NavigationRedirect) -> Value {
+ json!({
+ "from_url": redirect.from_url.as_str(),
+ "to_url": redirect.to_url.as_str(),
+ "status": redirect.status,
+ "headers": render_headers(&redirect.headers),
+ })
+}
+
pub(super) async fn render_html(
page: &mut Page,
strip: StripOptions,
diff --git a/moli/src/fetch_dump/mod.rs b/moli/src/fetch_dump/mod.rs
index 9247adc1d..6e4258696 100644
--- a/moli/src/fetch_dump/mod.rs
+++ b/moli/src/fetch_dump/mod.rs
@@ -72,8 +72,21 @@ pub fn render_raw_document_dump(
DumpFormat::Html => Ok(raw.body_bytes().to_vec()),
DumpFormat::Json => {
let html = String::from_utf8_lossy(raw.body_bytes());
- let payload =
- dom::render_json_payload(raw.final_url().as_str(), raw.status(), &html, None)?;
+ let redirect_chain = raw
+ .navigation_redirect_chain()
+ .iter()
+ .cloned()
+ .map(Into::into)
+ .collect::<Vec<_>>();
+ let payload = dom::render_json_payload(
+ raw.final_url().as_str(),
+ raw.status(),
+ None,
+ raw.headers(),
+ &redirect_chain,
+ &html,
+ None,
+ )?;
Ok(payload.into_bytes())
}
DumpFormat::Markdown
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index 3a6027370..08174815e 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -216,9 +216,25 @@ fn assert_json_dump_shape(payload: &Value, expected_url: &str, expected_status:
let mut keys = object.keys().cloned().collect::<Vec<_>>();
keys.sort();
- assert_eq!(keys, vec!["final_url", "html", "status"]);
+ assert_eq!(
+ keys,
+ vec![
+ "final_url",
+ "headers",
+ "html",
+ "redirect_chain",
+ "status",
+ "title",
+ ]
+ );
assert_eq!(payload["final_url"], expected_url);
assert_eq!(payload["status"], expected_status);
+ assert!(
+ payload["title"].is_string() || payload["title"].is_null(),
+ "payload={payload}"
+ );
+ assert!(payload["headers"].is_array(), "payload={payload}");
+ assert!(payload["redirect_chain"].is_array(), "payload={payload}");
assert!(payload["html"].is_string(), "payload={payload}");
}
@@ -2194,6 +2210,35 @@ fn cli_dump_json_reports_redirected_final_url() -> Result<()> {
Ok(())
}
+#[test]
+fn cli_dump_json_reports_redirect_chain_after_script_navigation() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/location-nav/http-redirect-source");
+ let redirect_url = server.url("/redirect");
+ let final_url = server.url("/static");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "moli fetch failed: stdout={}\nstderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &final_url, 200);
+ let redirects = payload["redirect_chain"]
+ .as_array()
+ .expect("redirect_chain must be an array");
+ assert_eq!(redirects.len(), 1, "stdout={stdout}");
+ assert_eq!(redirects[0]["from_url"], redirect_url);
+ assert_eq!(redirects[0]["to_url"], final_url);
+ assert_eq!(redirects[0]["status"], 307);
+ Ok(())
+}
+
#[test]
fn cli_domstable_waits_for_late_content() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
@@ -2731,6 +2776,61 @@ fn cli_http_redirect_with_location_does_not_consume_redirect_wait() -> Result<()
Ok(())
}
+#[test]
+fn cli_dump_json_includes_title_headers_and_redirect_chain() -> Result<()> {
+ let runtime = tokio::runtime::Runtime::new()?;
+ let server = runtime.block_on(FixtureServer::spawn())?;
+ let url = server.url("/redirect-cookie-chain/start");
+ let middle_url = server.url("/redirect-cookie-chain/middle");
+ let final_url = server.url("/redirect-cookie-chain/final");
+ let output = run_fetch_cli_with_dump_and_args(&url, "json", &[])?;
+ runtime.block_on(server.shutdown());
+
+ assert!(
+ output.status.success(),
+ "moli fetch failed: stdout={}\nstderr={}",
+ String::from_utf8_lossy(&output.stdout),
+ String::from_utf8_lossy(&output.stderr)
+ );
+ let stdout = clean_output(&output.stdout);
+ let payload: Value = serde_json::from_str(&stdout)?;
+ assert_json_dump_shape(&payload, &final_url, 200);
+ assert_eq!(payload["title"], "Cookie chain final");
+ assert!(
+ payload["headers"].as_array().is_some_and(|headers| {
+ headers.iter().any(|header| {
+ header["name"]
+ .as_str()
+ .is_some_and(|name| name.eq_ignore_ascii_case("content-type"))
+ })
+ }),
+ "stdout={stdout}"
+ );
+
+ let redirects = payload["redirect_chain"]
+ .as_array()
+ .expect("redirect_chain must be an array");
+ assert_eq!(redirects.len(), 2, "stdout={stdout}");
+ assert_eq!(redirects[0]["from_url"], url);
+ assert_eq!(redirects[0]["to_url"], middle_url);
+ assert_eq!(redirects[0]["status"], 307);
+ assert_eq!(redirects[1]["from_url"], middle_url);
+ assert_eq!(redirects[1]["to_url"], final_url);
+ assert_eq!(redirects[1]["status"], 307);
+ let first_hop_set_cookies = redirects[0]["headers"]
+ .as_array()
+ .expect("redirect headers must be an array")
+ .iter()
+ .filter(|header| {
+ header["name"]
+ .as_str()
+ .is_some_and(|name| name.eq_ignore_ascii_case("set-cookie"))
+ })
+ .count();
+ assert_eq!(first_hop_set_cookies, 2, "stdout={stdout}");
+ Ok(())
+}
+
#[test]
fn cli_domcontentloaded_recovers_from_delayed_403_navigation_at_same_stage() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
diff --git a/skills/moli-webfetch/SKILL.md b/skills/moli-webfetch/SKILL.md
index cdc55ff64..5896ac2ef 100644
--- a/skills/moli-webfetch/SKILL.md
+++ b/skills/moli-webfetch/SKILL.md
@@ -81,8 +81,9 @@ structure-first; enable layout only when the result needs pixels or pagination.
- Use `markdown` for prose, documentation, articles, and direct model reading.
- Use `semantic_tree_text` when navigation-heavy markup makes Markdown noisy or
when roles and accessible names matter.
-- Use `json` for automation that needs `final_url`, HTTP `status`, serialized
- `html`, or network trace data.
+- Use `json` for automation that needs `final_url`, HTTP `status`, `title`,
+ duplicate-safe response `headers`, the main-navigation `redirect_chain`,
+ serialized `html`, or network trace data.
- Use `html` to diagnose DOM serialization or preserve exact markup.
- Use `screenshot` for a viewport PNG when appearance is evidence. It requires
`--layout`.
diff --git a/skills/moli-webfetch/references/fetch-recipes.md b/skills/moli-webfetch/references/fetch-recipes.md
index 0c7bef55b..719bdcf2e 100644
--- a/skills/moli-webfetch/references/fetch-recipes.md
+++ b/skills/moli-webfetch/references/fetch-recipes.md
@@ -17,7 +17,7 @@
| Read page content | `--dump markdown` | Default for research and summarization |
| Inspect semantic structure | `--dump semantic_tree_text` | Compact roles, labels, text, and backend node IDs |
| Process semantic structure | `--dump semantic_tree` | Structured accessibility-oriented payload |
-| Process stable fields | `--dump json` | Returns `final_url`, `status`, and `html` |
+| Process stable fields | `--dump json` | Returns `final_url`, `status`, `title`, `headers`, `redirect_chain`, and `html` |
| Inspect exact DOM | `--dump html` | Useful when Markdown loses important structure |
| Diagnose requests | `--dump json --trace-network` | Adds the `network` object |
| Capture the viewport | `--layout --dump screenshot` | Writes PNG bytes to stdout |
@@ -26,6 +26,10 @@
Raw non-HTML responses support only `html` and `json`.
+JSON `headers` is an ordered list of `{name, value}` records so duplicate
+headers are preserved. `redirect_chain` contains every main-navigation HTTP
+redirect hop in order.
+
## Readiness
Start with `--wait-until done`. Change or extend it only when the page exposes