Lightpanda: the headless browser designed for AI and automation
indexeddb: serialize value at entry
Karl Seguin89c217c6104b
diff --git a/src/browser/js/Local.zig b/src/browser/js/Local.zigindex 76d8d2d36..528e50bbb 100644--- a/src/browser/js/Local.zig+++ b/src/browser/js/Local.zig@@ -1227,7 +1227,7 @@ fn jsIntToZig(comptime T: type, js_value: js.Value) !T {const v = js_value.toBigInt();return v.getInt64();}- return jsSignedIntToZig(i64, -2_147_483_648, 2_147_483_647, try js_value.toI32());+ return jsInt64ToZig(i64, try js_value.toF64());},else => {},},@@ -1250,7 +1250,7 @@ fn jsIntToZig(comptime T: type, js_value: js.Value) !T {const v = js_value.toBigInt();return v.getUint64();}- return jsUnsignedIntToZig(u64, 4_294_967_295, try js_value.toU32());+ return jsInt64ToZig(u64, try js_value.toF64());},else => {},},@@ -1272,6 +1272,20 @@ fn jsUnsignedIntToZig(comptime T: type, max: comptime_int, maybe: u32) !T {return error.InvalidArgument;}+// A JS number can't carry a full 64-bit integer, it needs to be represented as+// a f64 with fractions rounded towards zero.+fn jsInt64ToZig(comptime T: type, number: f64) !T {+ if (!std.math.isFinite(number)) {+ return error.InvalidArgument;+ }+ const truncated = @trunc(number);+ const min: f64 = if (@typeInfo(T).int.signedness == .signed) -std.math.maxInt(i54) else 0;+ if (truncated < min or truncated > std.math.maxInt(i54)) {+ return error.InvalidArgument;+ }+ return @intFromFloat(truncated);+}+// Every WebApi type has a class_id as T.JsApi.Meta.class_id. We use this to create// a JSValue class of the correct type. However, given a Node, we don't want// to create a Node class, we want to create a class of the most specific type.diff --git a/src/browser/js/js.zig b/src/browser/js/js.zigindex 0fbf1fe19..c366ce849 100644--- a/src/browser/js/js.zig+++ b/src/browser/js/js.zig@@ -317,13 +317,17 @@ pub fn simpleZigValueToJs(isolate: Isolate, value: anytype, comptime fail: bool,if (value >= 0 and value <= 4_294_967_295) {return @ptrCast(isolate.initInteger(@as(u32, @intCast(value))).handle);}- return @ptrCast(isolate.initBigInt(value).handle);+ if (value >= -2_147_483_648 and value < 0) {+ return @ptrCast(isolate.initInteger(@as(i32, @intCast(value))).handle);+ }+ // 64-bit numbers are represented as floats (that's how it works in JS)+ return @ptrCast(isolate.initNumber(@as(f64, @floatFromInt(value))).handle);},.comptime_int => {- if (value > -2_147_483_648 and value <= 4_294_967_295) {+ if (value >= -2_147_483_648 and value <= 4_294_967_295) {return @ptrCast(isolate.initInteger(value).handle);}- return @ptrCast(isolate.initBigInt(value).handle);+ return @ptrCast(isolate.initNumber(@as(f64, value)).handle);},.float, .comptime_float => return @ptrCast(isolate.initNumber(value).handle),.pointer => |ptr| {diff --git a/src/browser/tests/indexeddb.html b/src/browser/tests/indexeddb.htmlindex 973ebfd32..0b7e0a279 100644--- a/src/browser/tests/indexeddb.html+++ b/src/browser/tests/indexeddb.html@@ -1189,4 +1189,67 @@});}</script>-</body>+<!-- put()/add() structured-clone the value at call time: the record is the+ clone as of the call (later mutation of the argument is not stored), a+ generated key is injected into the clone rather than the caller's object,+ and an unserializable value throws DataCloneError synchronously. -->+<script id="write_clones_at_call_time" type=module>+{+ const state = await testing.async();+ const open = indexedDB.open("clone-at-call-db", 1);+ open.onupgradeneeded = (e) => {+ e.target.result.createObjectStore("s", { keyPath: "id", autoIncrement: true });+ };+ open.onsuccess = (e) => {+ const db = e.target.result;+ const tx = db.transaction("s", "readwrite");+ const store = tx.objectStore("s");+ const record = { data: "before" };+ store.put(record);+ record.data = "after";+ store.put(record); // same object again: still no key on it, so a 2nd record+ let threw = null;+ try {+ store.put({ id: 3, fn: () => {} });+ } catch (err) {+ threw = err.name;+ }+ tx.oncomplete = () => {+ const all = db.transaction("s").objectStore("s").getAll();+ all.onsuccess = () => state.resolve({ all: all.result, record, threw });+ };+ };+ await state.done((got) => {+ testing.expectEqual([{ data: "before", id: 1 }, { data: "after", id: 2 }], got.all);+ testing.expectEqual({ data: "after" }, got.record);+ testing.expectEqual("DataCloneError", got.threw);+ });+}+</script>++<!-- A version above 2^32 (Date.now() is a common choice) survives the round+ trip as a Number; out-of-range versions are a TypeError. -->+<script id="open_version_64bit" type=module>+{+ const state = await testing.async();+ const version = 1788161539190;+ const open = indexedDB.open("big-version-db", version);+ let upgrade = null;+ open.onupgradeneeded = (e) => {+ upgrade = { oldVersion: e.oldVersion, newVersion: e.newVersion };+ };+ open.onsuccess = (e) => {+ const db = e.target.result;+ state.resolve({ upgrade, version: db.version, type: typeof db.version });+ };+ await state.done((got) => {+ testing.expectEqual({ oldVersion: 0, newVersion: version }, got.upgrade);+ testing.expectEqual(version, got.version);+ testing.expectEqual("number", got.type);+ });+ for (const bad of [-1, 0.5, NaN, Infinity, "foo", 2 ** 53]) {+ testing.expectError("TypeError", () => indexedDB.open("big-version-db", bad));+ }+}+</script>+</body>\ No newline at end of filediff --git a/src/browser/webapi/storage/idb/IDBCursor.zig b/src/browser/webapi/storage/idb/IDBCursor.zigindex cbf8f25cd..816c4e04c 100644--- a/src/browser/webapi/storage/idb/IDBCursor.zig+++ b/src/browser/webapi/storage/idb/IDBCursor.zig@@ -250,8 +250,13 @@ pub fn update(self: *IDBCursor, value: js.Value, exec: *Execution) !*IDBRequest// Structured-clone the value now, synchronously: an unserializable value must// throw DataCloneError from update() itself, not fail later in the drain. The// stored clone also decouples the record from any later mutation of the arg.- const serialized = value.serialize() catch return error.TryCatchRethrow;- defer serialized.deinit();+ const serialized = blk: {+ self._txn._cloning = true;+ defer self._txn._cloning = false;+ break :blk value.serialize() catch return error.TryCatchRethrow;+ };+ const clone = try self._txn.holdClone(serialized);+ errdefer self._txn.releaseClone(clone);// For an in-line store, the value's own key must match the record's key.if (self._store._key_path) |kp| {@@ -262,28 +267,22 @@ pub fn update(self: *IDBCursor, value: js.Value, exec: *Execution) !*IDBRequest}}- // Snapshot the record key and the serialized clone onto the transaction arena:- // the write runs in the drain, by which point a `continue` could have moved- // the cursor's live position (and reused its key buffer).+ // Snapshot the record key onto the transaction arena: the write runs in the+ // drain, by which point a `continue` could have moved the cursor's live+ // position (and reused its key buffer).const key = try self._txn._arena.dupe(u8, current_key);- const bytes = try self._txn._arena.dupe(u8, serialized.bytes());const request = try self._txn.newRequest();- return request.submit(.{ .cursor_update = .{ .cursor = self, .key = key, .value = bytes } }, exec);+ return request.submit(.{ .cursor_update = .{ .cursor = self, .key = key, .value = clone } }, exec);}-pub fn runUpdate(self: *IDBCursor, request: *IDBRequest, key: []const u8, bytes: []const u8, exec: *Execution) !void {- const local = exec.js.local.?;- const value = js.Value.deserialize(local, bytes) catch |err| {- request.setError(err);- return;- };-- self._store.writeAt(key, value, bytes, exec) catch |err| {+pub fn runUpdate(self: *IDBCursor, request: *IDBRequest, key: []const u8, clone: usize, exec: *Execution) !void {+ defer self._txn.releaseClone(clone);+ self._store.writeAt(key, self._txn.cloneBytes(clone), exec) catch |err| {log.warn(.storage, "idb cursor update", .{ .err = err, .sqlite = self._engine.lastError() });request.setError(err);return;};- try request.setValue(try Key.decodeToJs(exec.call_arena, local, key));+ try request.setValue(try Key.decodeToJs(exec.call_arena, exec.js.local.?, key));}pub fn delete(self: *IDBCursor, exec: *Execution) !*IDBRequest {diff --git a/src/browser/webapi/storage/idb/IDBObjectStore.zig b/src/browser/webapi/storage/idb/IDBObjectStore.zigindex aec0747c6..104814df9 100644--- a/src/browser/webapi/storage/idb/IDBObjectStore.zig+++ b/src/browser/webapi/storage/idb/IDBObjectStore.zig@@ -308,15 +308,28 @@ fn write(self: *IDBObjectStore, value: js.Value, key_arg: ?js.Value, kind: Write}try txn.assertActive();+ if (self._key_path != null and key_arg != null) {+ // can't have an explicit key if we're configured for in-line keys+ return error.DataError;+ }++ // Structured-clone the value now, synchronously: an unserializable value+ // must throw DataCloneError from put()/add() itself, and the stored record+ // must not see mutations the caller makes after this returns. The write+ // itself runs in the drain, which releases the clone.+ const serialized = blk: {+ txn._cloning = true;+ defer txn._cloning = false;+ break :blk value.serialize() catch return error.TryCatchRethrow;+ };+ const clone = try txn.holdClone(serialized);+ errdefer txn.releaseClone(clone);+// Resolve (and validate) the key now, so DataError / a throwing key getter// throws synchronously. Encoded key bytes are captured on the transaction's- // arena to outlive this call. The record write itself is deferred to runWrite.+ // arena to outlive this call.const prepared: PreparedKey = blk: {if (self._key_path) |kp| {- if (key_arg != null) {- // can't have an explicit key if we're configured for in-line keys- return error.DataError;- }if (try Key.extractKeyPath(exec.js.local.?, value, kp)) |extracted| {break :blk .{ .explicit = .{.encoded = try Key.encodeValue(txn._arena.allocator(), extracted),@@ -349,20 +362,18 @@ fn write(self: *IDBObjectStore, value: js.Value, key_arg: ?js.Value, kind: Write};const request = try txn.newRequest();- const value_global = try txn.persist(value);return request.submit(.{ .store_write = .{.store = self,.kind = kind,- .value = value_global,+ .value = clone,.key = prepared,} }, exec);}-pub fn runWrite(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, value_global: *js.GlobalSlot, prepared: PreparedKey, exec: *Execution) !void {- // Written (or failed) is written: the pinned value is dead once this op- // ran, so release its handle now instead of at transaction teardown.- defer value_global.reset();- self.writeInner(request, kind, value_global, prepared, exec) catch |err| {+pub fn runWrite(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, clone: usize, prepared: PreparedKey, exec: *Execution) !void {+ // Written (or failed) is written: the clone is dead once this op ran.+ defer self._txn.releaseClone(clone);+ self.writeInner(request, kind, self._txn.cloneBytes(clone), prepared, exec) catch |err| {if (err != error.Constraint) {log.warn(.storage, "idb write", .{ .err = err, .kind = kind, .sqlite = self._engine.lastError() });}@@ -370,9 +381,14 @@ pub fn runWrite(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, va};}-fn writeInner(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, value_global: *js.GlobalSlot, prepared: PreparedKey, exec: *Execution) !void {+fn writeInner(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, stored: []const u8, prepared: PreparedKey, exec: *Execution) !void {const local = exec.js.local.?;- const value = value_global.local(local);++ // The bytes that hit the record, and the deserialized clone when one was+ // needed to inject a generated key (reindex otherwise deserializes lazily,+ // only if the store has indexes).+ var bytes = stored;+ var clone: ?js.Value = null;// Resolve the encoded key + the JS value that becomes the request result. For// generated keys, that's where the connection is finally touched.@@ -390,18 +406,21 @@ fn writeInner(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, valu.generate_in_line => blk: {const n = try self._engine.nextGeneratedKey(self._store_id);const k = try local.newNumber(@floatFromInt(n));- try Key.injectKey(local, value, self._key_path.?.string, k);+ // The key goes into the clone, never the caller's object.+ const v = try js.Value.deserialize(local, stored);+ try Key.injectKey(local, v, self._key_path.?.string, k);+ const reserialized = try v.serialize();+ defer reserialized.deinit();+ bytes = try exec.call_arena.dupe(u8, reserialized.bytes());+ clone = v;break :blk try Key.encodeValue(exec.call_arena, k);},};- const serialized = try value.serialize();- defer serialized.deinit();-// Record + index rows are atomic: a unique-index violation rolls the record// write back too.try self._engine.savepoint();- self.writeRecord(kind, encoded, serialized.bytes(), value, exec) catch |err| {+ self.writeRecord(kind, encoded, bytes, clone, exec) catch |err| {self._engine.rollbackSavepoint();return err;};@@ -411,21 +430,21 @@ fn writeInner(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, valutry request.setValue(try Key.decodeToJs(exec.call_arena, local, encoded));}-fn writeRecord(self: *IDBObjectStore, kind: WriteKind, key: []const u8, bytes: []const u8, value: js.Value, exec: *Execution) !void {+fn writeRecord(self: *IDBObjectStore, kind: WriteKind, key: []const u8, bytes: []const u8, clone: ?js.Value, exec: *Execution) !void {switch (kind) {.add => try self._engine.add(self._store_id, key, bytes),.put => try self._engine.put(self._store_id, key, bytes),}- try self.reindex(value, key, exec);+ try self.reindex(bytes, clone, key, exec);}// Used by IDBCursor.update: overwrite the record at `key` and re-index, atomically.-pub fn writeAt(self: *IDBObjectStore, key: []const u8, value: js.Value, bytes: []const u8, exec: *Execution) !void {+pub fn writeAt(self: *IDBObjectStore, key: []const u8, bytes: []const u8, exec: *Execution) !void {try self._engine.savepoint();errdefer self._engine.rollbackSavepoint();try self._engine.put(self._store_id, key, bytes);- try self.reindex(value, key, exec);+ try self.reindex(bytes, null, key, exec);try self._engine.releaseSavepoint();}@@ -437,12 +456,15 @@ pub fn deleteAt(self: *IDBObjectStore, key: []const u8) !void {}// Drop a record's old index entries and add fresh ones from `value`.-fn reindex(self: *IDBObjectStore, value: js.Value, primary_key: []const u8, exec: *Execution) !void {+// Index keys are extracted from the stored clone (`bytes`), which the caller+// passes already-deserialized as `clone` when it has it.+fn reindex(self: *IDBObjectStore, bytes: []const u8, clone: ?js.Value, primary_key: []const u8, exec: *Execution) !void {const arena = exec.call_arena;const indexes = try self._engine.indexesForStore(arena, self._store_id);if (indexes.len == 0) {return;}+ const value = clone orelse try js.Value.deserialize(exec.js.local.?, bytes);var seen: std.ArrayList([]const u8) = .empty;try self._engine.deleteIndexRecordsForKey(self._store_id, primary_key);diff --git a/src/browser/webapi/storage/idb/IDBRequest.zig b/src/browser/webapi/storage/idb/IDBRequest.zigindex 3f946c224..371305188 100644--- a/src/browser/webapi/storage/idb/IDBRequest.zig+++ b/src/browser/webapi/storage/idb/IDBRequest.zig@@ -372,11 +372,11 @@ pub const Operation = union(enum) {const StoreQuery = struct { store: *IDBObjectStore, bounds: Engine.Bounds };const StoreGetAll = struct { store: *IDBObjectStore, args: IDBKeyRange.GetAllArgs, mode: IDBObjectStore.GetAllMode };- const StoreWrite = struct { store: *IDBObjectStore, kind: IDBObjectStore.WriteKind, value: *js.GlobalSlot, key: IDBObjectStore.PreparedKey };+ const StoreWrite = struct { store: *IDBObjectStore, kind: IDBObjectStore.WriteKind, value: usize, key: IDBObjectStore.PreparedKey };const IndexQuery = struct { index: *IDBIndex, bounds: Engine.Bounds };const IndexGetAll = struct { index: *IDBIndex, args: IDBKeyRange.GetAllArgs, mode: IDBObjectStore.GetAllMode };const CursorIterate = struct { cursor: *IDBCursor, seek: IDBCursor.Seek, offset: u32 };- const CursorUpdate = struct { cursor: *IDBCursor, key: []const u8, value: []const u8 };+ const CursorUpdate = struct { cursor: *IDBCursor, key: []const u8, value: usize };const CursorDelete = struct { cursor: *IDBCursor, key: []const u8 };fn source(op: Operation) Source {diff --git a/src/browser/webapi/storage/idb/IDBTransaction.zig b/src/browser/webapi/storage/idb/IDBTransaction.zigindex f488a9a35..b350be355 100644--- a/src/browser/webapi/storage/idb/IDBTransaction.zig+++ b/src/browser/webapi/storage/idb/IDBTransaction.zig@@ -67,6 +67,10 @@ _arena: *lp.Arena,// a v8 Global reset is only idempotent through a single instance._globals: std.ArrayList(*js.GlobalSlot) = .empty,+// V8-owned data queued for a write. Only ever appended to, so an index into it+// is a stable handle (see holdClone).+_clones: std.ArrayList(CloneSlot) = .empty,+// objectStore() must return the same object for a given name within one// transaction (per spec); this also keeps repeated lookups off sqlite._stores: std.ArrayList(*IDBObjectStore) = .empty,@@ -87,6 +91,10 @@ _gate_waiter: Engine.GateWaiter,// capture the scheduler's generation here and reject any request made in a// later generation (see assertActive)._active_turn: u64 = 0,+// Set while a value is being structured-cloned for a write: user getters that+// run during the clone must see the transaction as inactive (spec: the clone+// happens with the transaction's active flag cleared).+_cloning: bool = false,// The transaction can be freed when v8 doesn't reference it (or any child,// e.g. an IDBRequest), when we have no scheduled drain AND when we aren't@@ -182,6 +190,9 @@ pub fn deinit(self: *IDBTransaction, _: *Page) void {for (self._globals.items) |slot| {slot.reset();}+ for (self._clones.items) |*slot| {+ slot.release();+ }self._arena.release();}@@ -203,6 +214,36 @@ pub fn persist(self: *IDBTransaction, value: js.Value) !*js.GlobalSlot {return slot;}+const CloneSlot = struct {+ // optional because it can be released twice, once after write and then+ // on deinit.+ _serialized: ?js.Value.Serialized,++ fn release(self: *CloneSlot) void {+ if (self._serialized) |serialized| {+ serialized.deinit();+ self._serialized = null;+ }+ }+};++// Wraps a js.Value.Serialized so that we're able to manage it. This not only+// avoids a copy, it means we don't have to (1) bloat the transaction's arena+// or (2) have a per-request arena+pub fn holdClone(self: *IDBTransaction, serialized: js.Value.Serialized) !usize {+ errdefer serialized.deinit();+ try self._clones.append(self._arena.allocator(), .{ ._serialized = serialized });+ return self._clones.items.len - 1;+}++pub fn cloneBytes(self: *const IDBTransaction, clone: usize) []const u8 {+ return self._clones.items[clone]._serialized.?.bytes();+}++pub fn releaseClone(self: *IDBTransaction, clone: usize) void {+ self._clones.items[clone].release();+}+pub fn dupe(self: *IDBTransaction, value: []const u8) ![]const u8 {if (lp.String.intern(value)) |v| {return v;@@ -341,7 +382,7 @@ fn commitAndComplete(self: *IDBTransaction, exec: *Execution) void {// the upgradeneeded dispatch for a versionchange transaction) and again by each// delivered batch.pub fn assertActive(self: *const IDBTransaction) !void {- if (self._settled or self._committing) {+ if (self._settled or self._committing or self._cloning) {return error.TransactionInactiveError;}if (self._active_turn != self._exec.js.scheduler.generation) {