Skip to content

The headless browser for AI agents and web scraping

Read supplied Response streams on demand

SGavrla1dd711965b0
diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js
index 1a66d14b..a7f75279 100644
--- a/crates/obscura-js/js/bootstrap.js
+++ b/crates/obscura-js/js/bootstrap.js
@@ -8498,6 +8498,27 @@ function _arrayBufferFromBytes(bytes) {
return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
}
+async function _readResponseStream(stream) {
+ const reader = stream.getReader();
+ const chunks = [];
+ let total = 0;
+ try {
+ while (true) {
+ const { value, done } = await reader.read();
+ if (done) break;
+ if (Object.prototype.toString.call(value) !== '[object Uint8Array]') {
+ throw new TypeError('Response body stream must contain Uint8Array chunks');
+ }
+ const chunk = value.slice();
+ chunks.push(chunk); total += chunk.byteLength;
+ }
+ } finally { reader.releaseLock(); }
+ const bytes = new Uint8Array(total);
+ let offset = 0;
+ for (const chunk of chunks) { bytes.set(chunk, offset); offset += chunk.byteLength; }
+ return bytes;
+}
+
function _installWasmStreamingFallback() {
if (typeof WebAssembly === 'undefined') return;
if (WebAssembly.instantiateStreaming && WebAssembly.instantiateStreaming.__obscuraFallback) return;
@@ -9183,7 +9204,10 @@ if (typeof Response === 'undefined') {
} else if (this.status === 204 || this.status === 205 || this.status === 304) {
body = null;
}
- this._bodyBytes = _bodyToUint8Array(body);
+ const stream = typeof ReadableStream === 'function' && body instanceof ReadableStream ? body : null;
+ if (stream && (stream.locked || stream._disturbed)) throw new TypeError('Body stream is locked or disturbed');
+ if (stream) this._streamSource = stream;
+ this._bodyBytes = stream ? new Uint8Array() : _bodyToUint8Array(body);
this.ok = this.status >= 200 && this.status < 300;
this.headers = new Headers(headers);
this.type = internal ? (init.type || 'default') : 'default';
@@ -9200,11 +9224,12 @@ if (typeof Response === 'undefined') {
this._bodyAbortReason = undefined;
}
_consumeBody() {
- if (this._bodyUsed) throw new TypeError("Body is already consumed");
+ if (this.bodyUsed || this._streamSource?.locked || this._bodyStream?.locked) throw new TypeError("Body is already consumed or locked");
this._bodyUsed = true;
if (this._bodyAborted) throw this._bodyAbortReason;
}
get body() {
+ if (this._streamSource) return this._streamSource;
if (this._bodyNull) return null;
if (this._bodyUsed) throw new TypeError("Body is already consumed");
if (!this._bodyStream) {
@@ -9225,15 +9250,16 @@ if (typeof Response === 'undefined') {
}
},
});
- // bodyUsed flips the moment the stream is locked for reading
- // (spec: the body becomes "disturbed"), which no state probe can
- // observe on a native ReadableStream, so hook getReader instead.
+ // Locking alone does not disturb a body; reading or cancelling does.
const response = this;
const stream = this._bodyStream;
const getReader = stream.getReader.bind(stream);
stream.getReader = function () {
- response._bodyUsed = true;
- return getReader();
+ const reader = getReader();
+ const read = reader.read.bind(reader), cancel = reader.cancel.bind(reader);
+ reader.read = function () { response._bodyUsed = true; return read(); };
+ reader.cancel = function (reason) { response._bodyUsed = true; return cancel(reason); };
+ return reader;
};
stream.releaseLock = function () {
response._bodyUsed = true;
@@ -9242,12 +9268,19 @@ if (typeof Response === 'undefined') {
}
return this._bodyStream;
}
- get bodyUsed() { return this._bodyUsed; }
- async text() { this._consumeBody(); return _decodeBodyWithCharset(this._fetchBody ? await this._fetchBody.promise : this._bodyBytes, this.headers); }
- async json() { this._consumeBody(); return JSON.parse(await _decodeBodyWithCharset(this._fetchBody ? await this._fetchBody.promise : this._bodyBytes, this.headers)); }
- async arrayBuffer() { this._consumeBody(); return _arrayBufferFromBytes(this._fetchBody ? await this._fetchBody.promise : this._bodyBytes); }
- async blob() { this._consumeBody(); return new Blob([this._fetchBody ? await this._fetchBody.promise : this._bodyBytes]); }
+ get bodyUsed() { return this._bodyUsed || !!this._streamSource?._disturbed; }
+ async text() { this._consumeBody(); return _decodeBodyWithCharset(this._streamSource ? await _readResponseStream(this._streamSource) : this._fetchBody ? await this._fetchBody.promise : this._bodyBytes, this.headers); }
+ async json() { this._consumeBody(); return JSON.parse(await _decodeBodyWithCharset(this._streamSource ? await _readResponseStream(this._streamSource) : this._fetchBody ? await this._fetchBody.promise : this._bodyBytes, this.headers)); }
+ async arrayBuffer() { this._consumeBody(); return _arrayBufferFromBytes(this._streamSource ? await _readResponseStream(this._streamSource) : this._fetchBody ? await this._fetchBody.promise : this._bodyBytes); }
+ async blob() { this._consumeBody(); return new Blob([this._streamSource ? await _readResponseStream(this._streamSource) : this._fetchBody ? await this._fetchBody.promise : this._bodyBytes]); }
clone() {
+ if (this.bodyUsed || this._streamSource?.locked || this._bodyStream?.locked) throw new TypeError('Body is already consumed or locked');
+ if (this._streamSource) {
+ const [left, right] = this._streamSource.tee();
+ this._streamSource = left;
+ return _createInternalResponse(right, {status: this.status, statusText: this.statusText,
+ headers: this.headers, type: this.type, url: this.url, redirected: this.redirected});
+ }
const copy = _createInternalResponse(this._bodyNull ? null : this._bodyBytes, { status: this.status, statusText: this.statusText, headers: this.headers, type: this.type, url: this.url, redirected: this.redirected });
if (this._fetchBody) {
const promise = this._fetchBody.promise.then(bytes => bytes.slice());
@@ -16881,6 +16914,12 @@ if (typeof ReadableStream === 'undefined') {
this._state = "readable";
this._error = null;
this.locked = false;
+ this._disturbed = false;
+ this._started = false;
+ this._pulling = false;
+ this._pullAgain = false;
+ this._highWaterMark = strategy.highWaterMark === undefined ? 1 : Number(strategy.highWaterMark);
+ if (Number.isNaN(this._highWaterMark) || this._highWaterMark < 0) throw new RangeError('Invalid highWaterMark');
const stream = this;
this._controller = {
enqueue(chunk) {
@@ -16888,6 +16927,7 @@ if (typeof ReadableStream === 'undefined') {
const pending = stream._reads.shift();
if (pending) pending.resolve({value: chunk, done: false});
else stream._queue.push(chunk);
+ stream._pullIfNeeded();
},
close() {
if (stream._state !== "readable") return;
@@ -16907,16 +16947,27 @@ if (typeof ReadableStream === 'undefined') {
while (stream._reads.length) stream._reads.shift().reject(error);
while (stream._closedWaiters.length) stream._closedWaiters.shift().reject(error);
},
- get desiredSize() { return Math.max(0, 1 - stream._queue.length); },
+ get desiredSize() { return stream._highWaterMark - stream._queue.length; },
};
try {
const started = source.start?.(this._controller);
if (started && typeof started.then === "function") {
- started.catch((error) => this._controller.error(error));
+ Promise.resolve(started).then(() => { this._started = true; this._pullIfNeeded(); }, (error) => this._controller.error(error));
+ } else {
+ this._started = true;
+ if (typeof source.pull === 'function') queueMicrotask(() => this._pullIfNeeded());
}
- } catch (error) {
- this._controller.error(error);
- }
+ } catch (error) { this._controller.error(error); }
+ }
+ _pullIfNeeded() {
+ if (!this._started || this._state !== 'readable' || this._closeRequested || typeof this._source.pull !== 'function'
+ || (!this._reads.length && this._controller.desiredSize <= 0)) return;
+ if (this._pulling) { this._pullAgain = true; return; }
+ this._pulling = true;
+ Promise.resolve().then(() => this._source.pull(this._controller)).then(() => {
+ this._pulling = false;
+ if (this._pullAgain) { this._pullAgain = false; this._pullIfNeeded(); }
+ }, error => { this._pulling = false; this._controller.error(error); });
}
getReader() {
if (this.locked) throw new TypeError("ReadableStream is locked");
@@ -16925,14 +16976,16 @@ if (typeof ReadableStream === 'undefined') {
let closedPromise;
return {
read() {
+ stream._disturbed = true;
if (stream._queue.length > 0) {
const value = stream._queue.shift();
if (!stream._queue.length && stream._closeRequested) stream._controller.close();
+ stream._pullIfNeeded();
return Promise.resolve({ value, done: false });
}
if (stream._state === "closed") return Promise.resolve({ value: undefined, done: true });
if (stream._state === "errored") return Promise.reject(stream._error);
- return new Promise((resolve, reject) => stream._reads.push({resolve, reject}));
+ return new Promise((resolve, reject) => { stream._reads.push({resolve, reject}); stream._pullIfNeeded(); });
},
releaseLock() { stream.locked = false; },
cancel(reason) { return stream.cancel(reason); },
@@ -16950,6 +17003,7 @@ if (typeof ReadableStream === 'undefined') {
};
}
cancel(reason) {
+ this._disturbed = true;
if (this._state === "errored") return Promise.reject(this._error);
this._queue.length = 0;
if (this._state === "closed") return Promise.resolve();
@@ -16982,27 +17036,43 @@ if (typeof ReadableStream === 'undefined') {
return transform.readable;
}
tee() {
- let leftController;
- let rightController;
- const left = new ReadableStream({start(controller) { leftController = controller; }});
- const right = new ReadableStream({start(controller) { rightController = controller; }});
- (async () => {
- try {
- const reader = this.getReader();
- while (true) {
- const {value, done} = await reader.read();
- if (done) break;
- leftController.enqueue(value);
- rightController.enqueue(value);
- }
- leftController.close();
- rightController.close();
- } catch (error) {
- leftController.error(error);
- rightController.error(error);
+ const reader = this.getReader(), controllers = [], cancelled = [false, false], reasons = [];
+ let reading = null, finished = false, completeCancel, rejectCancel;
+ const cancellation = new Promise((resolve, reject) => { completeCancel = resolve; rejectCancel = reject; });
+ cancellation.catch(() => {});
+ const errorBranches = error => {
+ if (finished) return;
+ finished = true;
+ for (let i = 0; i < 2; i++) if (!cancelled[i]) controllers[i].error(error);
+ reader.releaseLock(); completeCancel();
+ };
+ reader.closed.catch(errorBranches);
+ const pull = () => {
+ if (!reading && !finished) {
+ reading = reader.read().then(({value, done}) => {
+ if (finished) return;
+ if (done) {
+ finished = true;
+ for (let i = 0; i < 2; i++) if (!cancelled[i]) controllers[i].close();
+ reader.releaseLock(); completeCancel();
+ } else {
+ for (let i = 0; i < 2; i++) if (!cancelled[i]) controllers[i].enqueue(value);
+ }
+ }, errorBranches).finally(() => { reading = null; });
}
- })();
- return [left, right];
+ return reading;
+ };
+ return [0, 1].map(index => new ReadableStream({
+ start(controller) { controllers[index] = controller; }, pull,
+ cancel(reason) {
+ cancelled[index] = true; reasons[index] = reason;
+ if (cancelled[0] && cancelled[1]) {
+ finished = true;
+ Promise.resolve(reader.cancel(reasons)).then(completeCancel, rejectCancel).finally(() => reader.releaseLock());
+ }
+ return cancellation;
+ },
+ }));
}
[Symbol.asyncIterator]() {
const reader = this.getReader();
diff --git a/crates/obscura-js/tests/response_streams.rs b/crates/obscura-js/tests/response_streams.rs
new file mode 100644
index 00000000..04c15153
--- /dev/null
+++ b/crates/obscura-js/tests/response_streams.rs
@@ -0,0 +1,79 @@
+use obscura_js::runtime::ObscuraJsRuntime;
+use serde_json::{json, Value};
+
+async fn evaluate(source: &str) -> Value {
+ let mut runtime = ObscuraJsRuntime::new();
+ runtime.set_dom(obscura_dom::parse_html("<!doctype html><body></body>"));
+ runtime.run_page_init();
+ runtime.evaluate_for_cdp(source, true, true).await.unwrap().value.unwrap()
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn response_streams_preserve_bytes_errors_and_clone_consumption() {
+ let value = evaluate(r#"(async () => {
+ const original = new Response('{"ok":true}');
+ const json = await new Response(original.body).json();
+ const binary = new Response(new ReadableStream({start(c) { c.enqueue(new Uint8Array([0,255,128])); c.close(); }}));
+ const copy = binary.clone();
+ const unused = [binary.bodyUsed, copy.bodyUsed];
+ const a = [...new Uint8Array(await binary.arrayBuffer())];
+ const b = [...new Uint8Array(await copy.arrayBuffer())];
+ let badChunk = false, sourceError = false, consumed = false;
+ try { await new Response(new ReadableStream({start(c) {c.enqueue('not bytes');c.close();}})).text(); } catch(e) {badChunk=e instanceof TypeError;}
+ try { await new Response(new ReadableStream({start(c) {c.error(new Error('source failure'));}})).text(); } catch(e) {sourceError=e.message==='source failure';}
+ try { binary.clone(); } catch(e) { consumed=e instanceof TypeError; }
+ return {json,a,b,unused,badChunk,sourceError,consumed};
+ })()"#).await;
+ assert_eq!(value, json!({"json":{"ok":true},"a":[0,255,128],"b":[0,255,128],"unused":[false,false],"badChunk":true,"sourceError":true,"consumed":true}));
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn response_streams_keep_demand_and_cancellation_with_the_source() {
+ let value = evaluate(r#"(async () => {
+ let pulls=0, cancelled=null;
+ const source=new ReadableStream({pull(c){pulls++;c.enqueue(new Uint8Array([pulls]));},cancel(reason){cancelled=reason;}});
+ const response=new Response(source);
+ await new Promise(resolve=>setTimeout(resolve,0));
+ const lazy=pulls<=1, same=response.body===source, unused=!response.bodyUsed;
+ await response.body.cancel('stop');
+ const locked=new ReadableStream(),reader=locked.getReader();let rejected=false;
+ try{new Response(locked);}catch(e){rejected=e instanceof TypeError;}reader.releaseLock();
+ let count=0;
+ const bytes=await new Response(new ReadableStream({pull(c){count++;if(count<=3)c.enqueue(new Uint8Array([count]));else c.close();}})).arrayBuffer();
+ return {lazy,same,unused,cancelled,used:response.bodyUsed,rejected,bytes:[...new Uint8Array(bytes)]};
+ })()"#).await;
+ assert_eq!(value, json!({"lazy":true,"same":true,"unused":true,"cancelled":"stop","used":true,"rejected":true,"bytes":[1,2,3]}));
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn response_streams_reject_locked_bytes_and_forward_tee_failures() {
+ let value = evaluate(r#"(async () => {
+ const response=new Response('locked bytes'), reader=response.body.getReader();
+ let cloneRejected=false,textRejected=false;
+ try{response.clone();}catch(e){cloneRejected=e instanceof TypeError;}
+ try{await response.text();}catch(e){textRejected=e instanceof TypeError;}
+ reader.releaseLock();const text=await response.text();
+ let controller;const branches=new ReadableStream({start(c){controller=c;c.enqueue(new Uint8Array([1]));}}).tee();
+ await new Promise(resolve=>setTimeout(resolve,0));
+ const branch=branches[0].getReader();
+ const closed=branch.closed.then(()=>false,e=>e.message==='upstream error');
+ controller.error(new Error('upstream error'));const sourceClosed=await closed;let readError=false;
+ try{await branch.read();}catch(e){readError=e.message==='upstream error';}
+ const cancellations=new ReadableStream({cancel(){return Promise.reject(new Error('cancel failed'));}}).tee();
+ const settled=await Promise.allSettled(cancellations.map(b=>b.cancel()));
+ return {cloneRejected,textRejected,text,closed:sourceClosed,readError,cancelRejected:settled.every(s=>s.status==='rejected'&&s.reason.message==='cancel failed')};
+ })()"#).await;
+ assert_eq!(value, json!({"cloneRejected":true,"textRejected":true,"text":"locked bytes","closed":true,"readError":true,"cancelRejected":true}));
+}
+
+#[tokio::test(flavor = "current_thread")]
+async fn response_stream_clone_preserves_default_stream_chunk_identity() {
+ let value = evaluate(r#"(async () => {
+ const response = new Response(new ReadableStream({start(c) {c.enqueue(new Uint8Array([1])); c.close();}}));
+ const copy = response.clone();
+ const first = await response.body.getReader().read();
+ first.value[0] = 9;
+ return [...new Uint8Array(await copy.arrayBuffer())];
+ })()"#).await;
+ assert_eq!(value, json!([9]));
+}