Skip to content

Lightpanda: the headless browser designed for AI and automation

indexeddb: serialize value at entry

Karl Seguin89c217c6104b
diff --git a/src/browser/js/Local.zig b/src/browser/js/Local.zig
index 76d8d2d36..528e50bbb 100644
--- a/src/browser/js/Local.zig
+++ b/src/browser/js/Local.zig
@@ -1227,7 +1227,7 @@ fn jsIntToZig(comptime T: type, js_value: js.Value) !T {
const v = js_value.toBigInt();
return v.getInt64();
}
- return jsSignedIntToZig(i64, -2_147_483_648, 2_147_483_647, try js_value.toI32());
+ return jsInt64ToZig(i64, try js_value.toF64());
},
else => {},
},
@@ -1250,7 +1250,7 @@ fn jsIntToZig(comptime T: type, js_value: js.Value) !T {
const v = js_value.toBigInt();
return v.getUint64();
}
- return jsUnsignedIntToZig(u64, 4_294_967_295, try js_value.toU32());
+ return jsInt64ToZig(u64, try js_value.toF64());
},
else => {},
},
@@ -1272,6 +1272,20 @@ fn jsUnsignedIntToZig(comptime T: type, max: comptime_int, maybe: u32) !T {
return error.InvalidArgument;
}
+// A JS number can't carry a full 64-bit integer, it needs to be represented as
+// a f64 with fractions rounded towards zero.
+fn jsInt64ToZig(comptime T: type, number: f64) !T {
+ if (!std.math.isFinite(number)) {
+ return error.InvalidArgument;
+ }
+ const truncated = @trunc(number);
+ const min: f64 = if (@typeInfo(T).int.signedness == .signed) -std.math.maxInt(i54) else 0;
+ if (truncated < min or truncated > std.math.maxInt(i54)) {
+ return error.InvalidArgument;
+ }
+ return @intFromFloat(truncated);
+}
+
// Every WebApi type has a class_id as T.JsApi.Meta.class_id. We use this to create
// a JSValue class of the correct type. However, given a Node, we don't want
// to create a Node class, we want to create a class of the most specific type.
diff --git a/src/browser/js/js.zig b/src/browser/js/js.zig
index 0fbf1fe19..c366ce849 100644
--- a/src/browser/js/js.zig
+++ b/src/browser/js/js.zig
@@ -317,13 +317,17 @@ pub fn simpleZigValueToJs(isolate: Isolate, value: anytype, comptime fail: bool,
if (value >= 0 and value <= 4_294_967_295) {
return @ptrCast(isolate.initInteger(@as(u32, @intCast(value))).handle);
}
- return @ptrCast(isolate.initBigInt(value).handle);
+ if (value >= -2_147_483_648 and value < 0) {
+ return @ptrCast(isolate.initInteger(@as(i32, @intCast(value))).handle);
+ }
+ // 64-bit numbers are represented as floats (that's how it works in JS)
+ return @ptrCast(isolate.initNumber(@as(f64, @floatFromInt(value))).handle);
},
.comptime_int => {
- if (value > -2_147_483_648 and value <= 4_294_967_295) {
+ if (value >= -2_147_483_648 and value <= 4_294_967_295) {
return @ptrCast(isolate.initInteger(value).handle);
}
- return @ptrCast(isolate.initBigInt(value).handle);
+ return @ptrCast(isolate.initNumber(@as(f64, value)).handle);
},
.float, .comptime_float => return @ptrCast(isolate.initNumber(value).handle),
.pointer => |ptr| {
diff --git a/src/browser/tests/indexeddb.html b/src/browser/tests/indexeddb.html
index 973ebfd32..0b7e0a279 100644
--- a/src/browser/tests/indexeddb.html
+++ b/src/browser/tests/indexeddb.html
@@ -1189,4 +1189,67 @@
});
}
</script>
-</body>
+<!-- put()/add() structured-clone the value at call time: the record is the
+ clone as of the call (later mutation of the argument is not stored), a
+ generated key is injected into the clone rather than the caller's object,
+ and an unserializable value throws DataCloneError synchronously. -->
+<script id="write_clones_at_call_time" type=module>
+{
+ const state = await testing.async();
+ const open = indexedDB.open("clone-at-call-db", 1);
+ open.onupgradeneeded = (e) => {
+ e.target.result.createObjectStore("s", { keyPath: "id", autoIncrement: true });
+ };
+ open.onsuccess = (e) => {
+ const db = e.target.result;
+ const tx = db.transaction("s", "readwrite");
+ const store = tx.objectStore("s");
+ const record = { data: "before" };
+ store.put(record);
+ record.data = "after";
+ store.put(record); // same object again: still no key on it, so a 2nd record
+ let threw = null;
+ try {
+ store.put({ id: 3, fn: () => {} });
+ } catch (err) {
+ threw = err.name;
+ }
+ tx.oncomplete = () => {
+ const all = db.transaction("s").objectStore("s").getAll();
+ all.onsuccess = () => state.resolve({ all: all.result, record, threw });
+ };
+ };
+ await state.done((got) => {
+ testing.expectEqual([{ data: "before", id: 1 }, { data: "after", id: 2 }], got.all);
+ testing.expectEqual({ data: "after" }, got.record);
+ testing.expectEqual("DataCloneError", got.threw);
+ });
+}
+</script>
+
+<!-- A version above 2^32 (Date.now() is a common choice) survives the round
+ trip as a Number; out-of-range versions are a TypeError. -->
+<script id="open_version_64bit" type=module>
+{
+ const state = await testing.async();
+ const version = 1788161539190;
+ const open = indexedDB.open("big-version-db", version);
+ let upgrade = null;
+ open.onupgradeneeded = (e) => {
+ upgrade = { oldVersion: e.oldVersion, newVersion: e.newVersion };
+ };
+ open.onsuccess = (e) => {
+ const db = e.target.result;
+ state.resolve({ upgrade, version: db.version, type: typeof db.version });
+ };
+ await state.done((got) => {
+ testing.expectEqual({ oldVersion: 0, newVersion: version }, got.upgrade);
+ testing.expectEqual(version, got.version);
+ testing.expectEqual("number", got.type);
+ });
+ for (const bad of [-1, 0.5, NaN, Infinity, "foo", 2 ** 53]) {
+ testing.expectError("TypeError", () => indexedDB.open("big-version-db", bad));
+ }
+}
+</script>
+</body>
\ No newline at end of file
diff --git a/src/browser/webapi/storage/idb/IDBCursor.zig b/src/browser/webapi/storage/idb/IDBCursor.zig
index cbf8f25cd..816c4e04c 100644
--- a/src/browser/webapi/storage/idb/IDBCursor.zig
+++ b/src/browser/webapi/storage/idb/IDBCursor.zig
@@ -250,8 +250,13 @@ pub fn update(self: *IDBCursor, value: js.Value, exec: *Execution) !*IDBRequest
// Structured-clone the value now, synchronously: an unserializable value must
// throw DataCloneError from update() itself, not fail later in the drain. The
// stored clone also decouples the record from any later mutation of the arg.
- const serialized = value.serialize() catch return error.TryCatchRethrow;
- defer serialized.deinit();
+ const serialized = blk: {
+ self._txn._cloning = true;
+ defer self._txn._cloning = false;
+ break :blk value.serialize() catch return error.TryCatchRethrow;
+ };
+ const clone = try self._txn.holdClone(serialized);
+ errdefer self._txn.releaseClone(clone);
// For an in-line store, the value's own key must match the record's key.
if (self._store._key_path) |kp| {
@@ -262,28 +267,22 @@ pub fn update(self: *IDBCursor, value: js.Value, exec: *Execution) !*IDBRequest
}
}
- // Snapshot the record key and the serialized clone onto the transaction arena:
- // the write runs in the drain, by which point a `continue` could have moved
- // the cursor's live position (and reused its key buffer).
+ // Snapshot the record key onto the transaction arena: the write runs in the
+ // drain, by which point a `continue` could have moved the cursor's live
+ // position (and reused its key buffer).
const key = try self._txn._arena.dupe(u8, current_key);
- const bytes = try self._txn._arena.dupe(u8, serialized.bytes());
const request = try self._txn.newRequest();
- return request.submit(.{ .cursor_update = .{ .cursor = self, .key = key, .value = bytes } }, exec);
+ return request.submit(.{ .cursor_update = .{ .cursor = self, .key = key, .value = clone } }, exec);
}
-pub fn runUpdate(self: *IDBCursor, request: *IDBRequest, key: []const u8, bytes: []const u8, exec: *Execution) !void {
- const local = exec.js.local.?;
- const value = js.Value.deserialize(local, bytes) catch |err| {
- request.setError(err);
- return;
- };
-
- self._store.writeAt(key, value, bytes, exec) catch |err| {
+pub fn runUpdate(self: *IDBCursor, request: *IDBRequest, key: []const u8, clone: usize, exec: *Execution) !void {
+ defer self._txn.releaseClone(clone);
+ self._store.writeAt(key, self._txn.cloneBytes(clone), exec) catch |err| {
log.warn(.storage, "idb cursor update", .{ .err = err, .sqlite = self._engine.lastError() });
request.setError(err);
return;
};
- try request.setValue(try Key.decodeToJs(exec.call_arena, local, key));
+ try request.setValue(try Key.decodeToJs(exec.call_arena, exec.js.local.?, key));
}
pub fn delete(self: *IDBCursor, exec: *Execution) !*IDBRequest {
diff --git a/src/browser/webapi/storage/idb/IDBObjectStore.zig b/src/browser/webapi/storage/idb/IDBObjectStore.zig
index aec0747c6..104814df9 100644
--- a/src/browser/webapi/storage/idb/IDBObjectStore.zig
+++ b/src/browser/webapi/storage/idb/IDBObjectStore.zig
@@ -308,15 +308,28 @@ fn write(self: *IDBObjectStore, value: js.Value, key_arg: ?js.Value, kind: Write
}
try txn.assertActive();
+ if (self._key_path != null and key_arg != null) {
+ // can't have an explicit key if we're configured for in-line keys
+ return error.DataError;
+ }
+
+ // Structured-clone the value now, synchronously: an unserializable value
+ // must throw DataCloneError from put()/add() itself, and the stored record
+ // must not see mutations the caller makes after this returns. The write
+ // itself runs in the drain, which releases the clone.
+ const serialized = blk: {
+ txn._cloning = true;
+ defer txn._cloning = false;
+ break :blk value.serialize() catch return error.TryCatchRethrow;
+ };
+ const clone = try txn.holdClone(serialized);
+ errdefer txn.releaseClone(clone);
+
// Resolve (and validate) the key now, so DataError / a throwing key getter
// throws synchronously. Encoded key bytes are captured on the transaction's
- // arena to outlive this call. The record write itself is deferred to runWrite.
+ // arena to outlive this call.
const prepared: PreparedKey = blk: {
if (self._key_path) |kp| {
- if (key_arg != null) {
- // can't have an explicit key if we're configured for in-line keys
- return error.DataError;
- }
if (try Key.extractKeyPath(exec.js.local.?, value, kp)) |extracted| {
break :blk .{ .explicit = .{
.encoded = try Key.encodeValue(txn._arena.allocator(), extracted),
@@ -349,20 +362,18 @@ fn write(self: *IDBObjectStore, value: js.Value, key_arg: ?js.Value, kind: Write
};
const request = try txn.newRequest();
- const value_global = try txn.persist(value);
return request.submit(.{ .store_write = .{
.store = self,
.kind = kind,
- .value = value_global,
+ .value = clone,
.key = prepared,
} }, exec);
}
-pub fn runWrite(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, value_global: *js.GlobalSlot, prepared: PreparedKey, exec: *Execution) !void {
- // Written (or failed) is written: the pinned value is dead once this op
- // ran, so release its handle now instead of at transaction teardown.
- defer value_global.reset();
- self.writeInner(request, kind, value_global, prepared, exec) catch |err| {
+pub fn runWrite(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, clone: usize, prepared: PreparedKey, exec: *Execution) !void {
+ // Written (or failed) is written: the clone is dead once this op ran.
+ defer self._txn.releaseClone(clone);
+ self.writeInner(request, kind, self._txn.cloneBytes(clone), prepared, exec) catch |err| {
if (err != error.Constraint) {
log.warn(.storage, "idb write", .{ .err = err, .kind = kind, .sqlite = self._engine.lastError() });
}
@@ -370,9 +381,14 @@ pub fn runWrite(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, va
};
}
-fn writeInner(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, value_global: *js.GlobalSlot, prepared: PreparedKey, exec: *Execution) !void {
+fn writeInner(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, stored: []const u8, prepared: PreparedKey, exec: *Execution) !void {
const local = exec.js.local.?;
- const value = value_global.local(local);
+
+ // The bytes that hit the record, and the deserialized clone when one was
+ // needed to inject a generated key (reindex otherwise deserializes lazily,
+ // only if the store has indexes).
+ var bytes = stored;
+ var clone: ?js.Value = null;
// Resolve the encoded key + the JS value that becomes the request result. For
// generated keys, that's where the connection is finally touched.
@@ -390,18 +406,21 @@ fn writeInner(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, valu
.generate_in_line => blk: {
const n = try self._engine.nextGeneratedKey(self._store_id);
const k = try local.newNumber(@floatFromInt(n));
- try Key.injectKey(local, value, self._key_path.?.string, k);
+ // The key goes into the clone, never the caller's object.
+ const v = try js.Value.deserialize(local, stored);
+ try Key.injectKey(local, v, self._key_path.?.string, k);
+ const reserialized = try v.serialize();
+ defer reserialized.deinit();
+ bytes = try exec.call_arena.dupe(u8, reserialized.bytes());
+ clone = v;
break :blk try Key.encodeValue(exec.call_arena, k);
},
};
- const serialized = try value.serialize();
- defer serialized.deinit();
-
// Record + index rows are atomic: a unique-index violation rolls the record
// write back too.
try self._engine.savepoint();
- self.writeRecord(kind, encoded, serialized.bytes(), value, exec) catch |err| {
+ self.writeRecord(kind, encoded, bytes, clone, exec) catch |err| {
self._engine.rollbackSavepoint();
return err;
};
@@ -411,21 +430,21 @@ fn writeInner(self: *IDBObjectStore, request: *IDBRequest, kind: WriteKind, valu
try request.setValue(try Key.decodeToJs(exec.call_arena, local, encoded));
}
-fn writeRecord(self: *IDBObjectStore, kind: WriteKind, key: []const u8, bytes: []const u8, value: js.Value, exec: *Execution) !void {
+fn writeRecord(self: *IDBObjectStore, kind: WriteKind, key: []const u8, bytes: []const u8, clone: ?js.Value, exec: *Execution) !void {
switch (kind) {
.add => try self._engine.add(self._store_id, key, bytes),
.put => try self._engine.put(self._store_id, key, bytes),
}
- try self.reindex(value, key, exec);
+ try self.reindex(bytes, clone, key, exec);
}
// Used by IDBCursor.update: overwrite the record at `key` and re-index, atomically.
-pub fn writeAt(self: *IDBObjectStore, key: []const u8, value: js.Value, bytes: []const u8, exec: *Execution) !void {
+pub fn writeAt(self: *IDBObjectStore, key: []const u8, bytes: []const u8, exec: *Execution) !void {
try self._engine.savepoint();
errdefer self._engine.rollbackSavepoint();
try self._engine.put(self._store_id, key, bytes);
- try self.reindex(value, key, exec);
+ try self.reindex(bytes, null, key, exec);
try self._engine.releaseSavepoint();
}
@@ -437,12 +456,15 @@ pub fn deleteAt(self: *IDBObjectStore, key: []const u8) !void {
}
// Drop a record's old index entries and add fresh ones from `value`.
-fn reindex(self: *IDBObjectStore, value: js.Value, primary_key: []const u8, exec: *Execution) !void {
+// Index keys are extracted from the stored clone (`bytes`), which the caller
+// passes already-deserialized as `clone` when it has it.
+fn reindex(self: *IDBObjectStore, bytes: []const u8, clone: ?js.Value, primary_key: []const u8, exec: *Execution) !void {
const arena = exec.call_arena;
const indexes = try self._engine.indexesForStore(arena, self._store_id);
if (indexes.len == 0) {
return;
}
+ const value = clone orelse try js.Value.deserialize(exec.js.local.?, bytes);
var seen: std.ArrayList([]const u8) = .empty;
try self._engine.deleteIndexRecordsForKey(self._store_id, primary_key);
diff --git a/src/browser/webapi/storage/idb/IDBRequest.zig b/src/browser/webapi/storage/idb/IDBRequest.zig
index 3f946c224..371305188 100644
--- a/src/browser/webapi/storage/idb/IDBRequest.zig
+++ b/src/browser/webapi/storage/idb/IDBRequest.zig
@@ -372,11 +372,11 @@ pub const Operation = union(enum) {
const StoreQuery = struct { store: *IDBObjectStore, bounds: Engine.Bounds };
const StoreGetAll = struct { store: *IDBObjectStore, args: IDBKeyRange.GetAllArgs, mode: IDBObjectStore.GetAllMode };
- const StoreWrite = struct { store: *IDBObjectStore, kind: IDBObjectStore.WriteKind, value: *js.GlobalSlot, key: IDBObjectStore.PreparedKey };
+ const StoreWrite = struct { store: *IDBObjectStore, kind: IDBObjectStore.WriteKind, value: usize, key: IDBObjectStore.PreparedKey };
const IndexQuery = struct { index: *IDBIndex, bounds: Engine.Bounds };
const IndexGetAll = struct { index: *IDBIndex, args: IDBKeyRange.GetAllArgs, mode: IDBObjectStore.GetAllMode };
const CursorIterate = struct { cursor: *IDBCursor, seek: IDBCursor.Seek, offset: u32 };
- const CursorUpdate = struct { cursor: *IDBCursor, key: []const u8, value: []const u8 };
+ const CursorUpdate = struct { cursor: *IDBCursor, key: []const u8, value: usize };
const CursorDelete = struct { cursor: *IDBCursor, key: []const u8 };
fn source(op: Operation) Source {
diff --git a/src/browser/webapi/storage/idb/IDBTransaction.zig b/src/browser/webapi/storage/idb/IDBTransaction.zig
index f488a9a35..b350be355 100644
--- a/src/browser/webapi/storage/idb/IDBTransaction.zig
+++ b/src/browser/webapi/storage/idb/IDBTransaction.zig
@@ -67,6 +67,10 @@ _arena: *lp.Arena,
// a v8 Global reset is only idempotent through a single instance.
_globals: std.ArrayList(*js.GlobalSlot) = .empty,
+// V8-owned data queued for a write. Only ever appended to, so an index into it
+// is a stable handle (see holdClone).
+_clones: std.ArrayList(CloneSlot) = .empty,
+
// objectStore() must return the same object for a given name within one
// transaction (per spec); this also keeps repeated lookups off sqlite.
_stores: std.ArrayList(*IDBObjectStore) = .empty,
@@ -87,6 +91,10 @@ _gate_waiter: Engine.GateWaiter,
// capture the scheduler's generation here and reject any request made in a
// later generation (see assertActive).
_active_turn: u64 = 0,
+// Set while a value is being structured-cloned for a write: user getters that
+// run during the clone must see the transaction as inactive (spec: the clone
+// happens with the transaction's active flag cleared).
+_cloning: bool = false,
// The transaction can be freed when v8 doesn't reference it (or any child,
// e.g. an IDBRequest), when we have no scheduled drain AND when we aren't
@@ -182,6 +190,9 @@ pub fn deinit(self: *IDBTransaction, _: *Page) void {
for (self._globals.items) |slot| {
slot.reset();
}
+ for (self._clones.items) |*slot| {
+ slot.release();
+ }
self._arena.release();
}
@@ -203,6 +214,36 @@ pub fn persist(self: *IDBTransaction, value: js.Value) !*js.GlobalSlot {
return slot;
}
+const CloneSlot = struct {
+ // optional because it can be released twice, once after write and then
+ // on deinit.
+ _serialized: ?js.Value.Serialized,
+
+ fn release(self: *CloneSlot) void {
+ if (self._serialized) |serialized| {
+ serialized.deinit();
+ self._serialized = null;
+ }
+ }
+};
+
+// Wraps a js.Value.Serialized so that we're able to manage it. This not only
+// avoids a copy, it means we don't have to (1) bloat the transaction's arena
+// or (2) have a per-request arena
+pub fn holdClone(self: *IDBTransaction, serialized: js.Value.Serialized) !usize {
+ errdefer serialized.deinit();
+ try self._clones.append(self._arena.allocator(), .{ ._serialized = serialized });
+ return self._clones.items.len - 1;
+}
+
+pub fn cloneBytes(self: *const IDBTransaction, clone: usize) []const u8 {
+ return self._clones.items[clone]._serialized.?.bytes();
+}
+
+pub fn releaseClone(self: *IDBTransaction, clone: usize) void {
+ self._clones.items[clone].release();
+}
+
pub fn dupe(self: *IDBTransaction, value: []const u8) ![]const u8 {
if (lp.String.intern(value)) |v| {
return v;
@@ -341,7 +382,7 @@ fn commitAndComplete(self: *IDBTransaction, exec: *Execution) void {
// the upgradeneeded dispatch for a versionchange transaction) and again by each
// delivered batch.
pub fn assertActive(self: *const IDBTransaction) !void {
- if (self._settled or self._committing) {
+ if (self._settled or self._committing or self._cloning) {
return error.TransactionInactiveError;
}
if (self._active_turn != self._exec.js.scheduler.generation) {