Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(network): preserve curl completion order

ldm06566674ac7a1
diff --git a/moli-curl/src/runtime.rs b/moli-curl/src/runtime.rs
index 51af4007b..60c45050d 100644
--- a/moli-curl/src/runtime.rs
+++ b/moli-curl/src/runtime.rs
@@ -1,5 +1,4 @@
use std::{
- cmp::Reverse,
collections::VecDeque,
fmt,
num::NonZeroUsize,
@@ -557,8 +556,8 @@ impl<H: Handler + Send + 'static, C: Send + 'static> CurlRuntimeOwner<H, C> {
if let Err(error) = multi.perform() {
debug!("curl multi runtime perform failed: {error}");
}
- for (index, result) in completed_transfer_indices(multi, &state.active) {
- let active = state.active.swap_remove(index);
+ let completed = completed_transfer_indices(multi, &state.active);
+ for (active, result) in take_completed_in_notification_order(&mut state.active, completed) {
let easy = match multi.remove2(active.handle) {
Ok(easy) => Some(easy),
Err(error) => {
@@ -801,10 +800,35 @@ fn completed_transfer_indices<H: Handler, C>(
}
}
});
- completed.sort_by_key(|(index, _)| Reverse(*index));
completed
}
+/// Removes a completion batch without exposing the active-vector removal
+/// order to clients. `completed` is ordered exactly as libcurl reported its
+/// completion messages, while its indices refer to `active` before any member
+/// of the batch is removed.
+fn take_completed_in_notification_order<T, E>(
+ active: &mut Vec<T>,
+ completed: Vec<(usize, E)>,
+) -> Vec<(T, E)> {
+ let original_active_len = active.len();
+ let mut removed_indices = Vec::with_capacity(completed.len());
+ completed
+ .into_iter()
+ .map(|(original_index, result)| {
+ let removed_before = removed_indices
+ .iter()
+ .filter(|removed_index| **removed_index < original_index)
+ .count();
+ debug_assert!(original_index < original_active_len);
+ debug_assert!(!removed_indices.contains(&original_index));
+ let current_index = original_index - removed_before;
+ removed_indices.push(original_index);
+ (active.remove(current_index), result)
+ })
+ .collect()
+}
+
fn make_runtime_multi(config: &CurlMultiRuntimeConfig) -> Multi {
let mut multi = Multi::new();
if let Some(max_host_connections) = config.max_host_connections
@@ -911,6 +935,24 @@ mod tests {
assert_eq!(labels, ["high-a", "high-b", "auto-a", "auto-b", "low"]);
}
+ #[test]
+ fn completed_jobs_preserve_libcurl_notification_order() {
+ let mut active = vec!["first", "second", "third", "fourth"];
+
+ // libcurl reported the newest active entry first, followed by the
+ // oldest. Vector-index removal must not reverse that observable order.
+ let completed = take_completed_in_notification_order(
+ &mut active,
+ vec![(3, "fourth-result"), (0, "first-result")],
+ );
+
+ assert_eq!(
+ completed,
+ vec![("fourth", "fourth-result"), ("first", "first-result")]
+ );
+ assert_eq!(active, ["second", "third"]);
+ }
+
#[test]
fn per_origin_cap_blocks_only_matching_origin() {
let capped_origin = test_origin("example.test");
diff --git a/moli/tests/fetch_cli.rs b/moli/tests/fetch_cli.rs
index 7cc57df94..5b80b3a96 100644
--- a/moli/tests/fetch_cli.rs
+++ b/moli/tests/fetch_cli.rs
@@ -34,8 +34,8 @@ use support::FixtureServer;
use tokio::{io::AsyncReadExt, net::TcpListener, task::JoinHandle};
use tracing_subscriber::fmt::MakeWriter;
-#[path = "fetch_cli/anubis_async_module.rs"]
-mod anubis_async_module;
+#[path = "fetch_cli/anubis_deferred_module.rs"]
+mod anubis_deferred_module;
#[path = "fetch_cli/disable_js.rs"]
mod disable_js;
diff --git a/moli/tests/fetch_cli/anubis_async_module.rs b/moli/tests/fetch_cli/anubis_deferred_module.rs
similarity index 74%
rename from moli/tests/fetch_cli/anubis_async_module.rs
rename to moli/tests/fetch_cli/anubis_deferred_module.rs
index 7d1f0fd94..5b9463012 100644
--- a/moli/tests/fetch_cli/anubis_async_module.rs
+++ b/moli/tests/fetch_cli/anubis_deferred_module.rs
@@ -16,22 +16,22 @@ use std::{
};
use tokio::{net::TcpListener, sync::Notify, task::JoinHandle};
-// Reduced from Anubis 1.26.2 as served by lore.kernel.org. Its async module
-// suspends on a locale fetch while a later inline script is parser-blocked by
-// an earlier stylesheet. Once the stylesheet becomes available, the parser
-// must resume before a later fetch continuation can observe the following DOM.
+// Reduced from the Anubis 1.27 challenge template. Upstream changed the
+// challenge module from `async` to `defer` so it cannot observe DOM that the
+// stylesheet-blocked parser has not created yet.
const CHALLENGE_PAGE: &str = r#"<!doctype html>
<html>
<head>
<title>local anubis challenge</title>
<script>globalThis.__anubisOrder = ["bootstrap"];</script>
- <link rel="stylesheet" href="/anubis/blocked.css" onload="__anubisOrder.push('link-load'); fetch('/anubis/style-activated')">
+ <link rel="stylesheet" href="/anubis/blocked.css">
</head>
<body>
<main id="challenge">
<p id="status">calculating</p>
- <script async type="module" src="/anubis/main.mjs"></script>
+ <script defer type="module" src="/anubis/main.mjs"></script>
<script src="/anubis/parser-continuation.js"></script>
+ <div id="progress"><div class="bar-inner"></div></div>
<script>document.documentElement.dataset.executionOrder = __anubisOrder.join(',');</script>
</main>
</body>
@@ -42,11 +42,12 @@ const locale = await (await fetch('/anubis/locale.json')).json();
globalThis.__anubisOrder.push('module-continuation');
const progress = document.getElementById('progress');
document.documentElement.dataset.progressAtModuleContinuation = progress ? 'present' : 'missing';
-progress.style.display = 'block';
+if (progress) progress.style.display = 'block';
document.getElementById('status').textContent = locale.ready;
location.replace(
'/anubis/passed?order=' + encodeURIComponent(__anubisOrder.join(',')) +
- '&style=' + encodeURIComponent(document.documentElement.dataset.styleAtParserContinuation || 'missing')
+ '&style=' + encodeURIComponent(document.documentElement.dataset.styleAtParserContinuation || 'missing') +
+ '&progress=' + encodeURIComponent(document.documentElement.dataset.progressAtModuleContinuation)
);
"#;
@@ -55,10 +56,6 @@ globalThis.__anubisOrder.push('parser-continuation');
document.documentElement.dataset.parserContinuation = 'ran';
document.documentElement.dataset.styleAtParserContinuation =
getComputedStyle(document.getElementById('status')).color;
-const progress = document.createElement('div');
-progress.id = 'progress';
-progress.innerHTML = '<div class="bar-inner"></div>';
-document.currentScript.after(progress);
"#;
#[derive(Default)]
@@ -89,16 +86,14 @@ impl Event {
struct FixtureState {
stylesheet_requested: Arc<Event>,
parser_continuation_requested: Arc<Event>,
- locale_requested: Arc<Event>,
- style_activated: Arc<Event>,
}
-struct AnubisAsyncModuleFixture {
+struct AnubisDeferredModuleFixture {
base_url: String,
task: JoinHandle<()>,
}
-impl AnubisAsyncModuleFixture {
+impl AnubisDeferredModuleFixture {
async fn spawn() -> Result<Self> {
let state = FixtureState::default();
let listener = TcpListener::bind("127.0.0.1:0").await?;
@@ -111,14 +106,13 @@ impl AnubisAsyncModuleFixture {
get(|| async { javascript_response(MAIN_MODULE) }),
)
.route("/anubis/parser-continuation.js", get(parser_continuation))
- .route("/anubis/style-activated", get(style_activated))
.route("/anubis/locale.json", get(locale))
.route("/anubis/passed", get(passed))
.with_state(state);
let task = tokio::spawn(async move {
axum::serve(listener, app)
.await
- .expect("Anubis async-module fixture should serve");
+ .expect("Anubis deferred-module fixture should serve");
});
Ok(Self {
base_url: format!("http://{addr}"),
@@ -138,7 +132,7 @@ async fn passed(uri: Uri) -> Html<String> {
))
}
-impl Drop for AnubisAsyncModuleFixture {
+impl Drop for AnubisDeferredModuleFixture {
fn drop(&mut self) {
self.task.abort();
}
@@ -152,7 +146,6 @@ async fn blocked_stylesheet(State(state): State<FixtureState>) -> Response {
// the stylesheet terminal is released; the test does not depend on a
// sleep being long enough for the parser to get there.
state.parser_continuation_requested.wait().await;
- state.locale_requested.wait().await;
(
StatusCode::OK,
[
@@ -169,20 +162,8 @@ async fn parser_continuation(State(state): State<FixtureState>) -> Response {
javascript_response(PARSER_CONTINUATION)
}
-async fn style_activated(State(state): State<FixtureState>) -> StatusCode {
- state.style_activated.signal();
- StatusCode::NO_CONTENT
-}
-
async fn locale(State(state): State<FixtureState>) -> Response {
state.stylesheet_requested.wait().await;
- state.locale_requested.signal();
- // Keep the async module suspended until the stylesheet's actual DOM event
- // runs. The onload confirmation request is a browser-observed causal edge:
- // it avoids assuming that a server handler returning means response bytes
- // have already reached the renderer. If link-event work can overtake the
- // parser, releasing this response exposes the same Anubis failure.
- state.style_activated.wait().await;
(
StatusCode::OK,
[
@@ -207,9 +188,9 @@ fn javascript_response(source: &'static str) -> Response {
}
#[test]
-fn cli_async_module_does_not_overtake_stylesheet_unblocked_parser() -> Result<()> {
+fn cli_deferred_module_observes_stylesheet_unblocked_parser_dom() -> Result<()> {
let runtime = tokio::runtime::Runtime::new()?;
- let server = runtime.block_on(AnubisAsyncModuleFixture::spawn())?;
+ let server = runtime.block_on(AnubisDeferredModuleFixture::spawn())?;
let url = server.url("/anubis/challenge");
let output = Command::new(env!("CARGO_BIN_EXE_moli"))
.args([
@@ -236,15 +217,19 @@ fn cli_async_module_does_not_overtake_stylesheet_unblocked_parser() -> Result<()
);
assert!(
stdout.contains("id=\"passed\""),
- "async module resumed before the stylesheet-unblocked parser inserted #progress: stdout={stdout}\nstderr={stderr}"
+ "deferred module did not finish the local challenge: stdout={stdout}\nstderr={stderr}"
);
assert!(
- stdout.contains("order=bootstrap%2Cparser-continuation%2Clink-load%2Cmodule-continuation"),
- "stylesheet/parser ordering diverged from Chromium: stdout={stdout}\nstderr={stderr}"
+ stdout.contains("order=bootstrap%2Cparser-continuation%2Cmodule-continuation"),
+ "deferred module ran before parsing completed: stdout={stdout}\nstderr={stderr}"
);
assert!(
stdout.contains("style=rgb(1%2C%202%2C%203)"),
"the parser resumed before the completed stylesheet affected computed style: stdout={stdout}\nstderr={stderr}"
);
+ assert!(
+ stdout.contains("progress=present"),
+ "deferred module did not observe parser-created DOM: stdout={stdout}\nstderr={stderr}"
+ );
Ok(())
}