Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

feat(gateway): add keyset pagination and device member email (#6173)

Ezra Ellette0dfd675f92dc
diff --git a/crates/screenpipe-gateway/e2e/conformance/VENDORED_FROM b/crates/screenpipe-gateway/e2e/conformance/VENDORED_FROM
index aed33711e..4f350f642 100644
--- a/crates/screenpipe-gateway/e2e/conformance/VENDORED_FROM
+++ b/crates/screenpipe-gateway/e2e/conformance/VENDORED_FROM
@@ -1,7 +1,7 @@
repo: screenpipe/website (PRIVATE)
-branch: main
-commit: c3fc73b43c496d86fad54ecff65a4e350d179a7c
-date: 2026-08-11T16:43:39Z
+branch: codex/enterprise-v1-pagination
+commit: d2dd6427e9daff6b20aabedad1a211ee173a37a0
+date: 2026-08-12T19:05:55Z
files: openapi/enterprise-v1.yaml -> enterprise-v1.yaml
conformance/spec.ts -> spec.ts
conformance/cases.ts -> cases.ts
diff --git a/crates/screenpipe-gateway/e2e/conformance/cases.ts b/crates/screenpipe-gateway/e2e/conformance/cases.ts
index 24a0b5895..4b73abd5b 100644
--- a/crates/screenpipe-gateway/e2e/conformance/cases.ts
+++ b/crates/screenpipe-gateway/e2e/conformance/cases.ts
@@ -54,33 +54,42 @@ const FILES_W = { since: SEEDED_WINDOW.since };
export const cases: Case[] = [
// ── /devices ─────────────────────────────────────────────────────────────
{
- id: "devices: both seeded devices, newest first",
+ id: "devices: legacy shape and last-seen order remain unchanged",
path: "/api/enterprise/v1/devices",
expectStatus: 200,
expect: (b) => {
if (b.count !== EXPECTED.devices) throw new Error(`count=${b.count}`);
- if (b.devices.length !== b.count) throw new Error("count disagrees with devices.length");
+ if (b.devices.length !== b.count)
+ throw new Error("count disagrees with devices.length");
+ if ("next_cursor" in b)
+ throw new Error("legacy response grew next_cursor");
const ids = b.devices.map((d: any) => d.device_id).sort();
- if (JSON.stringify(ids) !== JSON.stringify([...EXPECTED.deviceIds].sort())) {
+ if (
+ JSON.stringify(ids) !== JSON.stringify([...EXPECTED.deviceIds].sort())
+ ) {
throw new Error(`device_ids=${JSON.stringify(ids)}`);
}
- // Ordering is part of the contract: last_seen DESC on both, so bob (hour
- // 10) precedes alice (hour 09) even though the two implementations derive
- // last_seen from different sources (DEV-LIVENESS).
if (b.devices[0].device_id !== "dev-bob") {
- throw new Error(`expected dev-bob first, got ${b.devices[0].device_id}`);
+ throw new Error(
+ `expected dev-bob first, got ${b.devices[0].device_id}`,
+ );
}
// Labels come from the wire format on the gateway and from the
// enterprise_devices row hosted; the fixtures make them agree.
- const byId = Object.fromEntries(b.devices.map((d: any) => [d.device_id, d]));
- if (byId["dev-alice"].label !== "alice-mbp") throw new Error("alice label");
- if (byId["dev-bob"].label !== "bob-thinkpad") throw new Error("bob label");
+ const byId = Object.fromEntries(
+ b.devices.map((d: any) => [d.device_id, d]),
+ );
+ if (byId["dev-alice"].label !== "alice-mbp")
+ throw new Error("alice label");
+ if (byId["dev-bob"].label !== "bob-thinkpad")
+ throw new Error("bob label");
},
expectHosted: {
why: "DEV-PLATFORM",
assert: (b) => {
const alice = b.devices.find((d: any) => d.device_id === "dev-alice");
- if (alice.platform !== "macos") throw new Error(`platform=${alice.platform}`);
+ if (alice.platform !== "macos")
+ throw new Error(`platform=${alice.platform}`);
},
},
expectGateway: {
@@ -88,12 +97,51 @@ export const cases: Case[] = [
assert: (b) => {
for (const d of b.devices) {
if (d.platform !== null || d.app_version !== null) {
- throw new Error(`gateway must report null platform/app_version, got ${JSON.stringify(d)}`);
+ throw new Error(
+ `gateway must report null platform/app_version, got ${JSON.stringify(d)}`,
+ );
}
}
},
},
},
+ {
+ id: "devices: member email is hosted identity and nullable on the gateway",
+ path: "/api/enterprise/v1/devices",
+ expectStatus: 200,
+ expectHosted: {
+ why: "DEV-MEMBER-EMAIL",
+ assert: (b) => {
+ const alice = b.devices.find((d: any) => d.device_id === "dev-alice");
+ if (alice.member_email !== "[email protected]") {
+ throw new Error(`member_email=${alice.member_email}`);
+ }
+ },
+ },
+ expectGateway: {
+ why: "DEV-MEMBER-EMAIL",
+ assert: (b) => {
+ for (const d of b.devices) {
+ if (d.member_email !== null) {
+ throw new Error(
+ `gateway must report null member_email, got ${JSON.stringify(d)}`,
+ );
+ }
+ }
+ },
+ },
+ },
+ {
+ id: "devices: limit=1 returns a continuation cursor",
+ path: "/api/enterprise/v1/devices",
+ query: { pagination: "keyset", limit: "1" },
+ expectStatus: 200,
+ expect: (b) => {
+ if (b.count !== 1) throw new Error(`limit ignored: ${b.count}`);
+ if (b.next_cursor === null)
+ throw new Error("second device was silently truncated");
+ },
+ },
// ── /search ──────────────────────────────────────────────────────────────
{
@@ -103,10 +151,16 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
if (b.query !== "roadmap") throw new Error(`query echoed as ${b.query}`);
- if (b.result_count !== b.results.length) throw new Error("result_count disagrees");
- if (b.result_count === 0) throw new Error("no results — the window or the corpus is wrong");
- if (b.window_hours !== 24) throw new Error(`window_hours=${b.window_hours}`);
- if (b.device_id !== null || b.app_name !== null) throw new Error("unfiltered echo");
+ if (b.result_count !== b.results.length)
+ throw new Error("result_count disagrees");
+ if ("next_cursor" in b)
+ throw new Error("legacy response grew next_cursor");
+ if (b.result_count === 0)
+ throw new Error("no results — the window or the corpus is wrong");
+ if (b.window_hours !== 24)
+ throw new Error(`window_hours=${b.window_hours}`);
+ if (b.device_id !== null || b.app_name !== null)
+ throw new Error("unfiltered echo");
const devices = new Set<string>(b.results.map((r: any) => r.device_id));
for (const id of EXPECTED.deviceIds) {
if (!devices.has(id)) {
@@ -116,7 +170,9 @@ export const cases: Case[] = [
const kinds = new Set<string>(b.results.map((r: any) => r.kind));
for (const k of ["frame", "parsed", "audio", "memory"]) {
if (!kinds.has(k)) {
- throw new Error(`missing kind ${k} in ${Array.from(kinds).join(", ")}`);
+ throw new Error(
+ `missing kind ${k} in ${Array.from(kinds).join(", ")}`,
+ );
}
}
// Every result must be INSIDE the requested window on both targets. The
@@ -127,33 +183,50 @@ export const cases: Case[] = [
throw new Error(`result outside the window: ${r.t}`);
}
}
+ const ts = b.results.map((r: any) => r.t);
+ if (JSON.stringify(ts) !== JSON.stringify([...ts].sort().reverse())) {
+ throw new Error(
+ `search results must be newest-first: ${JSON.stringify(ts)}`,
+ );
+ }
},
expectHosted: {
why: "SEARCH-ALGORITHM",
assert: (b) => {
- if (b.bytes_scanned <= 0) throw new Error("hosted must report a real byte count");
- if (b.objects_scanned <= 0) throw new Error("hosted must report scanned objects");
+ if (b.bytes_scanned <= 0)
+ throw new Error("hosted must report a real byte count");
+ if (b.objects_scanned <= 0)
+ throw new Error("hosted must report scanned objects");
},
},
expectGateway: {
why: "SEARCH-ALGORITHM",
assert: (b) => {
- if (b.bytes_scanned !== 0 || b.objects_scanned !== 0 || b.truncated !== false) {
+ if (
+ b.bytes_scanned !== 0 ||
+ b.objects_scanned !== 0 ||
+ b.truncated !== false
+ ) {
throw new Error(
"the gateway has no byte-budget scan to describe, so these must be " +
- `honestly 0/0/false — got ${b.bytes_scanned}/${b.objects_scanned}/${b.truncated}`
+ `honestly 0/0/false — got ${b.bytes_scanned}/${b.objects_scanned}/${b.truncated}`,
);
}
- // The gateway sorts results newest-first and the hosted API does not
- // (SEARCH-ORDER). Pinned here so the gateway's guarantee, which is the
- // stronger of the two, cannot silently regress.
- const ts = b.results.map((r: any) => r.t);
- if (JSON.stringify(ts) !== JSON.stringify([...ts].sort().reverse())) {
- throw new Error(`the gateway must sort newest-first: ${JSON.stringify(ts)}`);
- }
},
},
},
+ {
+ id: "search: limit=1 returns a continuation cursor",
+ path: "/api/enterprise/v1/search",
+ query: { q: "roadmap", ...W, pagination: "keyset", limit: "1" },
+ expectStatus: 200,
+ expect: (b) => {
+ if (b.result_count !== 1)
+ throw new Error(`limit ignored: ${b.result_count}`);
+ if (b.next_cursor === null)
+ throw new Error("matching records were silently truncated");
+ },
+ },
{
id: "search: device_id scopes every kind",
path: "/api/enterprise/v1/search",
@@ -161,9 +234,11 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
if (b.device_id !== "dev-alice") throw new Error(`echo=${b.device_id}`);
- if (b.result_count === 0) throw new Error("device filter returned nothing");
+ if (b.result_count === 0)
+ throw new Error("device filter returned nothing");
for (const r of b.results) {
- if (r.device_id !== "dev-alice") throw new Error(`leaked ${r.device_id}`);
+ if (r.device_id !== "dev-alice")
+ throw new Error(`leaked ${r.device_id}`);
}
},
},
@@ -173,7 +248,8 @@ export const cases: Case[] = [
query: { q: "roadmap", since: "2020-01-01", until: "2020-01-02" },
expectStatus: 200,
expect: (b) => {
- if (b.result_count !== 0 || b.results.length !== 0) throw new Error("expected empty");
+ if (b.result_count !== 0 || b.results.length !== 0)
+ throw new Error("expected empty");
},
},
{
@@ -184,10 +260,11 @@ export const cases: Case[] = [
expect: (b) => {
if (b.window_hours !== 24) {
throw new Error(
- `a bare date must be UTC midnight on both, giving a 24h window; got ${b.window_hours}`
+ `a bare date must be UTC midnight on both, giving a 24h window; got ${b.window_hours}`,
);
}
- if (b.result_count === 0) throw new Error("the July-22 window must contain the corpus");
+ if (b.result_count === 0)
+ throw new Error("the July-22 window must contain the corpus");
},
},
{
@@ -217,13 +294,20 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
if (b.kind !== "all") throw new Error(`kind echoed as ${b.kind}`);
- if (b.record_count !== b.records.length) throw new Error("record_count disagrees");
+ if (b.record_count !== b.records.length)
+ throw new Error("record_count disagrees");
if (b.record_count !== EXPECTED.recordsAllKinds) {
- throw new Error(`expected ${EXPECTED.recordsAllKinds} records, got ${b.record_count}`);
+ throw new Error(
+ `expected ${EXPECTED.recordsAllKinds} records, got ${b.record_count}`,
+ );
}
+ if ("next_cursor" in b)
+ throw new Error("legacy response grew next_cursor");
const ts = b.records.map((r: any) => r.t);
if (JSON.stringify(ts) !== JSON.stringify([...ts].sort())) {
- throw new Error(`records must be ascending by t: ${JSON.stringify(ts)}`);
+ throw new Error(
+ `records must be ascending by t: ${JSON.stringify(ts)}`,
+ );
}
},
},
@@ -233,7 +317,8 @@ export const cases: Case[] = [
query: { kind: "memory", ...W },
expectStatus: 200,
expect: (b) => {
- if (b.record_count !== EXPECTED.memories) throw new Error(`count=${b.record_count}`);
+ if (b.record_count !== EXPECTED.memories)
+ throw new Error(`count=${b.record_count}`);
for (const r of b.records) {
if (r.kind !== "memory") throw new Error(`leaked kind ${r.kind}`);
if (!r.content?.includes("gateway ships this quarter")) {
@@ -241,7 +326,9 @@ export const cases: Case[] = [
}
if (r.memory_id !== 1) throw new Error(`memory_id=${r.memory_id}`);
if (r.importance !== 0.8) throw new Error(`importance=${r.importance}`);
- if (JSON.stringify(r.tags) !== JSON.stringify(["decision", "roadmap"])) {
+ if (
+ JSON.stringify(r.tags) !== JSON.stringify(["decision", "roadmap"])
+ ) {
throw new Error(`tags=${JSON.stringify(r.tags)}`);
}
}
@@ -254,7 +341,8 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
if (b.kind !== "parsed") throw new Error(`kind=${b.kind}`);
- if (b.record_count !== EXPECTED.parsed) throw new Error(`count=${b.record_count}`);
+ if (b.record_count !== EXPECTED.parsed)
+ throw new Error(`count=${b.record_count}`);
for (const r of b.records) {
if (r.kind !== "parsed") throw new Error(`leaked kind ${r.kind}`);
if (r.app !== "Slack" || !r.text?.includes("structured update")) {
@@ -269,8 +357,10 @@ export const cases: Case[] = [
query: { kind: "nonsense", ...W },
expectStatus: 200,
expect: (b) => {
- if (b.kind !== "nonsense") throw new Error("the kind must still be echoed");
- if (b.record_count !== 0) throw new Error(`expected empty, got ${b.record_count}`);
+ if (b.kind !== "nonsense")
+ throw new Error("the kind must still be echoed");
+ if (b.record_count !== 0)
+ throw new Error(`expected empty, got ${b.record_count}`);
},
},
{
@@ -282,16 +372,20 @@ export const cases: Case[] = [
// It is a case anyway because it pins the SHAPE of the empty answer, which
// is what a consumer of the documented divergence relies on.
expect: (b) => {
- if (b.record_count !== 0) throw new Error(`expected empty, got ${b.record_count}`);
+ if (b.record_count !== 0)
+ throw new Error(`expected empty, got ${b.record_count}`);
},
},
{
id: "records: limit is respected",
path: "/api/enterprise/v1/records",
- query: { ...W, limit: "3" },
+ query: { ...W, pagination: "keyset", limit: "3" },
expectStatus: 200,
expect: (b) => {
- if (b.record_count !== 3) throw new Error(`limit ignored: ${b.record_count}`);
+ if (b.record_count !== 3)
+ throw new Error(`limit ignored: ${b.record_count}`);
+ if (b.next_cursor === null)
+ throw new Error("remaining records were silently truncated");
},
},
@@ -302,8 +396,10 @@ export const cases: Case[] = [
query: FILES_W,
expectStatus: 200,
expect: (b) => {
- if (b.count !== b.files.length) throw new Error("count disagrees with files.length");
- if (b.count !== EXPECTED.batchObjects) throw new Error(`count=${b.count}`);
+ if (b.count !== b.files.length)
+ throw new Error("count disagrees with files.length");
+ if (b.count !== EXPECTED.batchObjects)
+ throw new Error(`count=${b.count}`);
for (const f of b.files) {
if (!f.key.startsWith(`enterprise-telemetry/${LICENSE_ID}/`)) {
throw new Error(`key escapes the org prefix: ${f.key}`);
@@ -321,7 +417,8 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
if (b.count !== 1) throw new Error(`page_size ignored: ${b.count}`);
- if (b.next_cursor === null) throw new Error("more objects exist, so next_cursor must be set");
+ if (b.next_cursor === null)
+ throw new Error("more objects exist, so next_cursor must be set");
},
},
{
@@ -331,7 +428,8 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
// This is THE case: 1 item hosted, 100 on the gateway, before the fix.
- if (b.count !== 1) throw new Error(`expected the clamp to 1, got ${b.count}`);
+ if (b.count !== 1)
+ throw new Error(`expected the clamp to 1, got ${b.count}`);
},
},
{
@@ -350,7 +448,8 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
if (b.device_id !== "dev-alice") throw new Error(`echo=${b.device_id}`);
- if (b.count !== 1) throw new Error(`expected 1 alice object, got ${b.count}`);
+ if (b.count !== 1)
+ throw new Error(`expected 1 alice object, got ${b.count}`);
if (b.files[0].device_id !== "dev-alice") throw new Error("wrong device");
},
},
@@ -397,10 +496,16 @@ export const cases: Case[] = [
if (b.device !== "org") throw new Error(`device=${b.device}`);
if (b.count !== b.rollups.length) throw new Error("count disagrees");
if (b.count !== EXPECTED.rollups) throw new Error(`count=${b.count}`);
+ if ("next_cursor" in b)
+ throw new Error("legacy response grew next_cursor");
const r = b.rollups[0];
if (r.day !== "2026-07-22") throw new Error(`day=${r.day}`);
- if (r.data.records !== 12) throw new Error(`data passed through wrong: ${JSON.stringify(r.data)}`);
- if (JSON.stringify(r.data.devices) !== JSON.stringify(["dev-alice", "dev-bob"])) {
+ if (r.data.records !== 12)
+ throw new Error(`data passed through wrong: ${JSON.stringify(r.data)}`);
+ if (
+ JSON.stringify(r.data.devices) !==
+ JSON.stringify(["dev-alice", "dev-bob"])
+ ) {
throw new Error(`data.devices=${JSON.stringify(r.data.devices)}`);
}
},
@@ -412,7 +517,8 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
if (b.count !== 0) throw new Error(`expected empty, got ${b.count}`);
- if (b.from !== "2020-01-01" || b.to !== "2020-01-02") throw new Error("bounds not echoed");
+ if (b.from !== "2020-01-01" || b.to !== "2020-01-02")
+ throw new Error("bounds not echoed");
},
},
{
@@ -426,7 +532,8 @@ export const cases: Case[] = [
// because the intuitive expectation ("dev___etc") is wrong, and because a
// surviving ".." only fails to be traversal since the value is a storage
// key prefix rather than a filesystem path.
- if (b.device !== "dev_.._etc") throw new Error(`sanitized to ${b.device}`);
+ if (b.device !== "dev_.._etc")
+ throw new Error(`sanitized to ${b.device}`);
},
},
{
@@ -448,7 +555,8 @@ export const cases: Case[] = [
expectStatus: 200,
expect: (b) => {
if (b.kind !== "sop") throw new Error(`kind=${b.kind}`);
- if (b.count !== b.entries.length) throw new Error("count disagrees with entries");
+ if (b.count !== b.entries.length)
+ throw new Error("count disagrees with entries");
if (b.count !== 1) throw new Error(`count=${b.count}`);
},
},
@@ -459,7 +567,8 @@ export const cases: Case[] = [
targets: ["hosted"],
expectStatus: 400,
expect: (b) => {
- if (!b.error?.includes("invalid kind")) throw new Error(`error=${b.error}`);
+ if (!b.error?.includes("invalid kind"))
+ throw new Error(`error=${b.error}`);
// Every v1 400 carries a machine code. This one did not until the spec
// declared BadRequest for it and the suite refused to validate the body —
// which is the whole point of writing the contract down first.
diff --git a/crates/screenpipe-gateway/e2e/conformance/enterprise-v1.yaml b/crates/screenpipe-gateway/e2e/conformance/enterprise-v1.yaml
index 2db4bf354..42ec4ebe5 100644
--- a/crates/screenpipe-gateway/e2e/conformance/enterprise-v1.yaml
+++ b/crates/screenpipe-gateway/e2e/conformance/enterprise-v1.yaml
@@ -80,10 +80,29 @@ x-divergences:
Neither field is in the wire format, so the gateway has no source for
them. Typed `[string, "null"]` on both rather than omitted, so a consumer
writes one branch instead of two.
+ DEV-MEMBER-EMAIL:
+ id: DEV-MEMBER-EMAIL
+ operations: ["GET /devices"]
+ fields: ["devices[].member_email"]
+ hosted: >-
+ enterprise_devices.member_email from the latest device heartbeat: the
+ verified account email when member-authenticated, otherwise null.
+ gateway: always null
+ rationale: >-
+ The customer-run gateway reads telemetry batches from object storage and
+ has no account-identity channel. Keeping the field required and nullable
+ gives consumers one stable response shape without inventing identity.
SEARCH-ALGORITHM:
id: SEARCH-ALGORITHM
operations: ["GET /search"]
- fields: ["results", "records_scanned", "bytes_scanned", "objects_scanned", "truncated"]
+ fields:
+ [
+ "results",
+ "records_scanned",
+ "bytes_scanned",
+ "objects_scanned",
+ "truncated",
+ ]
hosted: >-
case-insensitive SUBSTRING match over a 256KB byte budget. `q=oadma`
matches "roadmap". The three counters describe that scan and `truncated`
@@ -97,26 +116,6 @@ x-divergences:
Ingest-once is the gateway's reason to exist; throwing away the index to
emulate a substring scan over blobs would make it slower AND worse. The
counters are honestly zero rather than fabricated.
- SEARCH-ORDER:
- id: SEARCH-ORDER
- operations: ["GET /search"]
- fields: ["results"]
- hosted: >-
- results come back in SCAN order: objects newest-first, but lines in file
- order within each object. So a single object's frame/parsed/audio/ui/memory rows
- appear in the order the device wrote them, not by timestamp.
- gateway: >-
- results are sorted by `t` DESCENDING before truncation.
- rationale: >-
- NOT ruled on — found by this suite, not by the SCR-288 audit, and recorded
- rather than silently converged because changing hosted result ordering is a
- consumer-visible behavior change that needs an owner. The gateway's
- guarantee is the stronger one and is pinned by the suite so it cannot
- regress; the shared assertion is the weaker property both hold (every
- result is inside the requested window). `/records` is NOT affected — both
- sort ascending by `t` there.
- TODO(owner): decide whether hosted /search should sort. If yes this entry
- goes away and the assertion moves into the shared `expect`.
SNAPSHOT-KIND:
id: SNAPSHOT-KIND
operations: ["GET /records"]
@@ -139,8 +138,8 @@ x-divergences:
a page can come back count:0 with a non-null next_cursor. The cursor is
the storage backend's continuation token.
gateway: >-
- time-filters the whole listing then applies skip/take, so `count` is exact
- and `next_cursor` is a decimal offset.
+ time-filters the whole listing then applies a stable object-key keyset, so
+ `count` is exact and inserts before the cursor do not shift later pages.
rationale: >-
Cursors are OPAQUE by contract and are not interchangeable across targets.
A client must treat next_cursor as a token from the same target and must
@@ -193,13 +192,21 @@ paths:
operationId: listDevices
summary: List the devices enrolled under the bearer's organization
x-scope: read:devices
- x-divergence: [DEV-LIVENESS, DEV-PLATFORM, TIMESTAMP-SERIALIZATION]
+ x-divergence:
+ [DEV-LIVENESS, DEV-PLATFORM, DEV-MEMBER-EMAIL, TIMESTAMP-SERIALIZATION]
+ parameters:
+ - $ref: "#/components/parameters/Pagination"
+ - $ref: "#/components/parameters/Limit"
+ - $ref: "#/components/parameters/Cursor"
responses:
"200":
- description: The organization's devices, most recently seen first
+ description: >-
+ Devices in legacy last-seen order by default, or stable device-id
+ order when pagination=keyset.
content:
application/json:
schema: { $ref: "#/components/schemas/DeviceList" }
+ "400": { $ref: "#/components/responses/BadRequest" }
"401": { $ref: "#/components/responses/Unauthorized" }
"403": { $ref: "#/components/responses/Forbidden" }
"500": { $ref: "#/components/responses/ServerError" }
@@ -209,7 +216,7 @@ paths:
operationId: searchRecords
summary: Search telemetry records
x-scope: read:search
- x-divergence: [SEARCH-ALGORITHM, SEARCH-ORDER]
+ x-divergence: [SEARCH-ALGORITHM]
parameters:
- $ref: "#/components/parameters/Q"
- $ref: "#/components/parameters/DeviceId"
@@ -217,7 +224,9 @@ paths:
- $ref: "#/components/parameters/Since"
- $ref: "#/components/parameters/Until"
- $ref: "#/components/parameters/SinceHoursAgo"
+ - $ref: "#/components/parameters/Pagination"
- $ref: "#/components/parameters/Limit"
+ - $ref: "#/components/parameters/Cursor"
responses:
"200":
description: Matching records, newest first
@@ -251,7 +260,9 @@ paths:
- $ref: "#/components/parameters/Since"
- $ref: "#/components/parameters/Until"
- $ref: "#/components/parameters/SinceHoursAgo"
+ - $ref: "#/components/parameters/Pagination"
- $ref: "#/components/parameters/Limit"
+ - $ref: "#/components/parameters/Cursor"
responses:
"200":
description: Records in the window, oldest first
@@ -389,6 +400,8 @@ paths:
in: query
required: false
schema: { type: integer, default: 35, minimum: 1, maximum: 100 }
+ - $ref: "#/components/parameters/Pagination"
+ - $ref: "#/components/parameters/Cursor"
responses:
"200":
description: Rollups in range, newest day first
@@ -583,6 +596,24 @@ components:
in: query
required: false
schema: { type: integer, default: 50, minimum: 1, maximum: 200 }
+ Pagination:
+ name: pagination
+ in: query
+ required: false
+ schema: { type: string, enum: [keyset] }
+ description: >-
+ Set to `keyset` to opt into pagination and receive `next_cursor`.
+ Omit it for the legacy response shape and ordering.
+ Cursor:
+ name: cursor
+ in: query
+ required: false
+ schema: { type: string }
+ description: >-
+ OPAQUE continuation token from a previous response's `next_cursor`.
+ Pass it back unchanged with the same filters; clients must not construct it.
+ Search, records, devices, and rollups require `pagination=keyset` and
+ use keyset state tied to a frozen scan boundary.
responses:
BadRequest:
@@ -706,7 +737,12 @@ components:
app: { type: [string, "null"] }
window: { type: [string, "null"] }
url: { type: [string, "null"] }
- text: { type: [string, "null"], description: Frame OCR / accessibility text, or parsed-app projection text }
+ text:
+ {
+ type: [string, "null"],
+ description: Frame OCR / accessibility text,
+ or parsed-app projection text,
+ }
transcription: { type: [string, "null"] }
speaker: { type: [string, "null"] }
content: { type: [string, "null"], description: 'kind:"memory" only' }
@@ -721,12 +757,36 @@ components:
Device:
type: object
additionalProperties: false
- required: [device_id, label, platform, app_version, last_seen, enrolled_at]
+ required:
+ [
+ device_id,
+ label,
+ member_email,
+ platform,
+ app_version,
+ last_seen,
+ enrolled_at,
+ ]
properties:
device_id: { type: string }
- label: { type: string, description: Hostname when known, else the device id }
- platform: { type: [string, "null"], description: Always null on the gateway (DEV-PLATFORM) }
- app_version: { type: [string, "null"], description: Always null on the gateway (DEV-PLATFORM) }
+ label:
+ { type: string, description: Hostname when known, else the device id }
+ member_email:
+ type: [string, "null"]
+ format: email
+ description: >-
+ Verified account email carried by the latest member-authenticated
+ heartbeat; always null on the gateway (DEV-MEMBER-EMAIL).
+ platform:
+ {
+ type: [string, "null"],
+ description: Always null on the gateway (DEV-PLATFORM),
+ }
+ app_version:
+ {
+ type: [string, "null"],
+ description: Always null on the gateway (DEV-PLATFORM),
+ }
last_seen:
type: [string, "null"]
format: date-time
@@ -745,6 +805,9 @@ components:
devices:
type: array
items: { $ref: "#/components/schemas/Device" }
+ next_cursor:
+ type: [string, "null"]
+ description: Present only with pagination=keyset; null on the final page.
SearchResult:
type: object
@@ -769,13 +832,34 @@ components:
minimum: 0
description: Effective response window; use the `since_hours_ago` query parameter to request a relative window.
records_scanned: { type: integer, minimum: 0 }
- bytes_scanned: { type: integer, minimum: 0, description: Always 0 on the gateway (SEARCH-ALGORITHM) }
- objects_scanned: { type: integer, minimum: 0, description: Always 0 on the gateway (SEARCH-ALGORITHM) }
- truncated: { type: boolean, description: Always false on the gateway (SEARCH-ALGORITHM) }
+ bytes_scanned:
+ {
+ type: integer,
+ minimum: 0,
+ description: Always 0 on the gateway (SEARCH-ALGORITHM),
+ }
+ objects_scanned:
+ {
+ type: integer,
+ minimum: 0,
+ description: Always 0 on the gateway (SEARCH-ALGORITHM),
+ }
+ truncated:
+ {
+ type: boolean,
+ description: Always false on the gateway (SEARCH-ALGORITHM),
+ }
result_count: { type: integer, minimum: 0 }
results:
type: array
items: { $ref: "#/components/schemas/RecordSummary" }
+ next_cursor:
+ type: [string, "null"]
+ description: >-
+ Present only with pagination=keyset. Mutation-safe token for more matches
+ in the frozen scan window. A null
+ cursor with truncated:true means the byte budget, not pagination, ended
+ the scan; narrow the query or time window.
RecordList:
type: object
@@ -791,7 +875,12 @@ components:
- records
properties:
device_id: { type: [string, "null"] }
- kind: { type: string, description: Echoed back lower-cased, including an unrecognized one }
+ kind:
+ {
+ type: string,
+ description: Echoed back lower-cased,
+ including an unrecognized one,
+ }
window_hours:
type: number
minimum: 0
@@ -803,6 +892,13 @@ components:
records:
type: array
items: { $ref: "#/components/schemas/RecordSummary" }
+ next_cursor:
+ type: [string, "null"]
+ description: >-
+ Present only with pagination=keyset. Mutation-safe token for more records
+ in the frozen scan window. A null
+ cursor with truncated:true means the byte budget, not pagination, ended
+ the scan; narrow the time window.
FileEntry:
type: object
@@ -810,12 +906,21 @@ components:
required: [key, size, last_modified, device_id]
properties:
key: { type: string }
- size: { type: [integer, "null"], minimum: 0, description: Nullable on the gateway (FILES-NULLABILITY) }
+ size:
+ {
+ type: [integer, "null"],
+ minimum: 0,
+ description: Nullable on the gateway (FILES-NULLABILITY),
+ }
last_modified:
type: [string, "null"]
format: date-time
description: Nullable on the gateway (FILES-NULLABILITY)
- device_id: { type: string, description: '"unknown" when the key has no device segment' }
+ device_id:
+ {
+ type: string,
+ description: '"unknown" when the key has no device segment',
+ }
FileList:
type: object
@@ -864,6 +969,9 @@ components:
rollups:
type: array
items: { $ref: "#/components/schemas/Rollup" }
+ next_cursor:
+ type: [string, "null"]
+ description: Present only with pagination=keyset; null on the final page.
ManifestEntry:
type: object
@@ -883,7 +991,8 @@ components:
- key
properties:
artifact_id: { type: string }
- kind: { type: string, enum: [sop, skill, memory, chart, trajectory, intel] }
+ kind:
+ { type: string, enum: [sop, skill, memory, chart, trajectory, intel] }
type: { type: string }
title: { type: string }
status: { type: string }
@@ -925,7 +1034,12 @@ components:
additionalProperties: false
required: [name, prompt_body]
properties:
- name: { type: string, pattern: "^[a-zA-Z0-9][a-zA-Z0-9_-]*$", maxLength: 100 }
+ name:
+ {
+ type: string,
+ pattern: "^[a-zA-Z0-9][a-zA-Z0-9_-]*$",
+ maxLength: 100,
+ }
prompt_body: { type: string, maxLength: 262144 }
display_name: { type: string }
schedule: { type: string, default: every 30m }
diff --git a/crates/screenpipe-gateway/e2e/conformance/fixtures/devices.json b/crates/screenpipe-gateway/e2e/conformance/fixtures/devices.json
index df39db583..b432daa57 100644
--- a/crates/screenpipe-gateway/e2e/conformance/fixtures/devices.json
+++ b/crates/screenpipe-gateway/e2e/conformance/fixtures/devices.json
@@ -9,8 +9,10 @@
"gateway (divergence DEV-PLATFORM): neither field is in the wire format, so",
"there is nothing for a gateway to read them from. `last_seen_at` is",
"heartbeat liveness here and MAX(record timestamp) there (DEV-LIVENESS).",
- "Both are recorded divergences, so the suite asserts their TYPE and not",
- "their value.",
+ "`member_email` is the hosted heartbeat identity and is ALWAYS null on the",
+ "gateway (DEV-MEMBER-EMAIL), which has no account-identity channel.",
+ "The contract keeps every field present and nullable; target-specific cases",
+ "pin the values available on each side.",
"",
"`last_seen_at` sits inside the seeded July-22 window so a hosted /devices",
"response is ordered the same way a gateway one is (bob newer than alice)."
@@ -19,6 +21,7 @@
{
"device_id": "dev-bob",
"hostname": "bob-thinkpad",
+ "member_email": "[email protected]",
"platform": "windows",
"app_version": "2.5.136",
"last_seen_at": "2026-07-22T10:08:00+00:00",
@@ -27,6 +30,7 @@
{
"device_id": "dev-alice",
"hostname": "alice-mbp",
+ "member_email": "[email protected]",
"platform": "macos",
"app_version": "2.5.136",
"last_seen_at": "2026-07-22T09:08:00+00:00",
diff --git a/crates/screenpipe-gateway/e2e/conformance/fixtures/index.ts b/crates/screenpipe-gateway/e2e/conformance/fixtures/index.ts
index 534c4b938..b5e698f05 100644
--- a/crates/screenpipe-gateway/e2e/conformance/fixtures/index.ts
+++ b/crates/screenpipe-gateway/e2e/conformance/fixtures/index.ts
@@ -47,6 +47,7 @@ export const NON_SK_ENT_TOKEN: string = tokensFile.non_sk_ent_token;
export const DEVICE_ROWS: Array<{
device_id: string;
hostname: string;
+ member_email: string | null;
platform: string;
app_version: string;
last_seen_at: string;
diff --git a/crates/screenpipe-gateway/e2e/conformance/run.test.ts b/crates/screenpipe-gateway/e2e/conformance/run.test.ts
index f7b5a4e69..a971959ba 100644
--- a/crates/screenpipe-gateway/e2e/conformance/run.test.ts
+++ b/crates/screenpipe-gateway/e2e/conformance/run.test.ts
@@ -21,22 +21,31 @@
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
-vi.mock("@/lib/supabase", async () => (await import("./targets/hosted-inproc")).supabaseMock());
+vi.mock("@/lib/supabase", async () =>
+ (await import("./targets/hosted-inproc")).supabaseMock(),
+);
vi.mock("@/lib/enterprise/storage", async () =>
- (await import("./targets/hosted-inproc")).storageMock()
+ (await import("./targets/hosted-inproc")).storageMock(),
+);
+vi.mock("@/lib/enterprise/audit", async () =>
+ (await import("./targets/hosted-inproc")).auditMock(),
);
-vi.mock("@/lib/enterprise/audit", async () => (await import("./targets/hosted-inproc")).auditMock());
vi.mock("@/lib/enterprise/artifacts/store", async () =>
- (await import("./targets/hosted-inproc")).artifactsStoreMock()
+ (await import("./targets/hosted-inproc")).artifactsStoreMock(),
);
vi.mock("@/lib/enterprise/auth", async () =>
- (await import("./targets/hosted-inproc")).enterpriseAuthMock()
+ (await import("./targets/hosted-inproc")).enterpriseAuthMock(),
);
vi.mock("@clerk/backend", () => ({ verifyToken: vi.fn() }));
vi.mock("@clerk/nextjs/server", () => ({ clerkClient: vi.fn() }));
import { cases, type Case } from "./cases";
-import { ALL_SCOPES_TOKEN, EXPECTED, LICENSE_ID, SEEDED_WINDOW } from "./fixtures";
+import {
+ ALL_SCOPES_TOKEN,
+ EXPECTED,
+ LICENSE_ID,
+ SEEDED_WINDOW,
+} from "./fixtures";
import {
contentTypeFor,
operationFor,
@@ -58,7 +67,9 @@ beforeAll(() => {
process.env.SUPABASE_SERVICE_KEY ||= "conformance-service-key";
});
-async function send(req: Parameters<typeof import("./targets/http").request>[0]) {
+async function send(
+ req: Parameters<typeof import("./targets/http").request>[0],
+) {
if (activeTarget() === "http") {
return (await import("./targets/http")).request(req);
}
@@ -72,7 +83,8 @@ async function send(req: Parameters<typeof import("./targets/http").request>[0])
* catch-all routes are templated in the spec and concrete in a case.
*/
function specPathFor(path: string): string {
- if (path.startsWith("/api/enterprise/v1/files/")) return "/api/enterprise/v1/files/{key}";
+ if (path.startsWith("/api/enterprise/v1/files/"))
+ return "/api/enterprise/v1/files/{key}";
if (path.startsWith("/api/enterprise/v1/frames/")) {
return "/api/enterprise/v1/frames/{device_id}/{frame_id}";
}
@@ -101,20 +113,25 @@ async function check(c: Case) {
const raw = await res.clone().text();
expect(
res.status,
- `${method} ${c.path} -> ${res.status}, body: ${raw.slice(0, 400)}`
+ `${method} ${c.path} -> ${res.status}, body: ${raw.slice(0, 400)}`,
).toBe(c.expectStatus);
// 2. Content type, exactly as the spec declares it for this status.
const declared = contentTypeFor(method, specPath, c.expectStatus);
const actual = (res.headers.get("content-type") ?? "").split(";")[0].trim();
- expect(actual, `${method} ${c.path} ${c.expectStatus} content-type`).toBe(declared);
+ expect(actual, `${method} ${c.path} ${c.expectStatus} content-type`).toBe(
+ declared,
+ );
// 3. Required response headers.
for (const [name, schema] of Object.entries<any>(
- requiredHeaders(method, specPath, c.expectStatus)
+ requiredHeaders(method, specPath, c.expectStatus),
)) {
const value = res.headers.get(name);
- expect(value, `${method} ${c.path}: required header '${name}' is missing`).not.toBeNull();
+ expect(
+ value,
+ `${method} ${c.path}: required header '${name}' is missing`,
+ ).not.toBeNull();
if (schema.schema?.const !== undefined) {
expect(value, `header '${name}'`).toBe(schema.schema.const);
}
@@ -129,7 +146,7 @@ async function check(c: Case) {
expect(
ok,
`${method} ${c.path} ${c.expectStatus} does not match the spec:\n${validate.errorText()}\n` +
- `body: ${JSON.stringify(body).slice(0, 600)}`
+ `body: ${JSON.stringify(body).slice(0, 600)}`,
).toBe(true);
}
@@ -140,7 +157,9 @@ async function check(c: Case) {
return { res, body, raw };
}
-const applicable = cases.filter((c) => (c.targets ?? ["hosted", "gateway"]).includes(impl));
+const applicable = cases.filter((c) =>
+ (c.targets ?? ["hosted", "gateway"]).includes(impl),
+);
describe(`conformance [${activeTarget()} / impl=${impl}]`, () => {
it("has cases to run (an empty run must never look like a pass)", () => {
@@ -154,6 +173,106 @@ describe(`conformance [${activeTarget()} / impl=${impl}]`, () => {
}
});
+describe(`keyset pagination [${impl}]`, () => {
+ it.each([
+ [
+ "devices",
+ "/api/enterprise/v1/devices",
+ { pagination: "keyset", limit: "1" },
+ "devices",
+ ],
+ [
+ "search",
+ "/api/enterprise/v1/search",
+ {
+ q: "roadmap",
+ since: SEEDED_WINDOW.since,
+ until: SEEDED_WINDOW.until,
+ pagination: "keyset",
+ limit: "1",
+ },
+ "results",
+ ],
+ [
+ "records",
+ "/api/enterprise/v1/records",
+ {
+ since: SEEDED_WINDOW.since,
+ until: SEEDED_WINDOW.until,
+ pagination: "keyset",
+ limit: "3",
+ },
+ "records",
+ ],
+ [
+ "files",
+ "/api/enterprise/v1/files",
+ { since: SEEDED_WINDOW.since, page_size: "1" },
+ "files",
+ ],
+ ])(
+ "%s follows the returned opaque cursor without repeating a row",
+ async (name, path, query, field) => {
+ const first = await send({ path, query, headers: authHeader() });
+ expect(first.status, await first.clone().text()).toBe(200);
+ const firstBody = await first.json();
+ expect(firstBody.next_cursor).toEqual(expect.any(String));
+ if (name !== "files" || impl === "gateway") {
+ expect(firstBody.next_cursor).not.toMatch(/^\d+$/);
+ }
+
+ const second = await send({
+ path,
+ query: { ...query, cursor: firstBody.next_cursor },
+ headers: authHeader(),
+ });
+ expect(second.status, await second.clone().text()).toBe(200);
+ const secondBody = await second.json();
+ const identity = (row: any) =>
+ row.kind === undefined
+ ? (row.key ?? row.device_id)
+ : [
+ row.device_id,
+ row.kind,
+ row.t,
+ row.frame_id,
+ row.memory_id,
+ row.text,
+ row.transcription,
+ ].join("|");
+ expect(firstBody[field].map(identity)).not.toContain(
+ secondBody[field][0] && identity(secondBody[field][0]),
+ );
+ },
+ );
+
+ it("binds a cursor to its original filters", async () => {
+ const first = await send({
+ path: "/api/enterprise/v1/search",
+ query: {
+ q: "roadmap",
+ ...SEEDED_WINDOW,
+ pagination: "keyset",
+ limit: "1",
+ },
+ headers: authHeader(),
+ });
+ const body = await first.json();
+ const reused = await send({
+ path: "/api/enterprise/v1/search",
+ query: {
+ q: "different",
+ ...SEEDED_WINDOW,
+ pagination: "keyset",
+ limit: "1",
+ cursor: body.next_cursor,
+ },
+ headers: authHeader(),
+ });
+ expect(reused.status).toBe(400);
+ });
+});
+
/**
* The two binary routes need a key/id discovered from a previous response, so
* they cannot be static table rows. Their 404 and validation cases ARE in the
@@ -171,9 +290,14 @@ describe(`binary routes [${impl}]`, () => {
expect(files.length).toBe(EXPECTED.batchObjects);
const key: string = files[0].key;
- const res = await send({ path: `/api/enterprise/v1/files/${key}`, headers: authHeader() });
+ const res = await send({
+ path: `/api/enterprise/v1/files/${key}`,
+ headers: authHeader(),
+ });
expect(res.status, await res.clone().text()).toBe(200);
- expect((res.headers.get("content-type") ?? "").split(";")[0]).toBe("application/x-ndjson");
+ expect((res.headers.get("content-type") ?? "").split(";")[0]).toBe(
+ "application/x-ndjson",
+ );
// The spec marks this header required — it is how a consumer knows WHICH
// object it got when it followed a cursor.
expect(res.headers.get("x-screenpipe-source")).toBe(key);
@@ -183,7 +307,9 @@ describe(`binary routes [${impl}]`, () => {
const lines = text.trim().split("\n");
expect(lines.length).toBe(7); // the six in-window records + one stale record
const kinds = lines.map((l) => JSON.parse(l).kind);
- expect(new Set(kinds)).toEqual(new Set(["frame", "parsed", "audio", "ui", "memory"]));
+ expect(new Set(kinds)).toEqual(
+ new Set(["frame", "parsed", "audio", "ui", "memory"]),
+ );
for (const line of lines) {
const rec = JSON.parse(line);
expect(EXPECTED.deviceIds).toContain(rec.device_id);
@@ -202,12 +328,14 @@ describe(`binary routes [${impl}]`, () => {
headers: authHeader(),
});
expect(res.status, await res.clone().text()).toBe(200);
- expect((res.headers.get("content-type") ?? "").split(";")[0]).toBe("image/jpeg");
+ expect((res.headers.get("content-type") ?? "").split(";")[0]).toBe(
+ "image/jpeg",
+ );
expect(res.headers.get("cache-control")).toBe("private, max-age=3600");
const bytes = new Uint8Array(await res.arrayBuffer());
// A real JPEG SOI marker, so a JSON error body cannot pass as an image.
expect([bytes[0], bytes[1]]).toEqual([0xff, 0xd8]);
- }
+ },
);
});
@@ -218,43 +346,62 @@ describe(`binary routes [${impl}]`, () => {
* This is the SCR-288 divergence-11 fix, asserted at the surface rather than the
* unit: every content scope refuses, and the two hand-rolled routes refuse too.
*/
-describe.runIf(activeTarget() === "hosted-inproc")("write-only archive (hosted only)", () => {
- beforeEach(async () => {
- const { posture } = await import("./targets/hosted-inproc");
- posture.writeOnlyArchive = true;
- });
+describe.runIf(activeTarget() === "hosted-inproc")(
+ "write-only archive (hosted only)",
+ () => {
+ beforeEach(async () => {
+ const { posture } = await import("./targets/hosted-inproc");
+ posture.writeOnlyArchive = true;
+ });
- const contentRoutes: Array<[string, string, Record<string, string> | undefined]> = [
- ["GET", "/api/enterprise/v1/search", { q: "roadmap" }],
- ["GET", "/api/enterprise/v1/records", undefined],
- ["GET", "/api/enterprise/v1/files", undefined],
- ["GET", "/api/enterprise/v1/rollups", undefined],
- ["GET", "/api/enterprise/v1/workflows/generated", undefined],
- [
- "GET",
- `/api/enterprise/v1/files/enterprise-telemetry/${LICENSE_ID}/dev-alice/direct/batch-alice.jsonl`,
- undefined,
- ],
- ["GET", "/api/enterprise/v1/frames/dev-alice/1", undefined],
- ];
+ const contentRoutes: Array<
+ [string, string, Record<string, string> | undefined]
+ > = [
+ ["GET", "/api/enterprise/v1/search", { q: "roadmap" }],
+ ["GET", "/api/enterprise/v1/records", undefined],
+ ["GET", "/api/enterprise/v1/files", undefined],
+ ["GET", "/api/enterprise/v1/rollups", undefined],
+ ["GET", "/api/enterprise/v1/workflows/generated", undefined],
+ [
+ "GET",
+ `/api/enterprise/v1/files/enterprise-telemetry/${LICENSE_ID}/dev-alice/direct/batch-alice.jsonl`,
+ undefined,
+ ],
+ ["GET", "/api/enterprise/v1/frames/dev-alice/1", undefined],
+ ];
- it.each(contentRoutes)("%s %s refuses with a typed 409", async (method, path, query) => {
- const { request } = await import("./targets/hosted-inproc");
- const res = await request({ method, path, query, headers: authHeader() });
- const raw = await res.clone().text();
- expect(res.status, `${path} -> ${res.status}: ${raw.slice(0, 300)}`).toBe(409);
- const validate = validatorFor(method, specPathFor(path), 409)!;
- const body = JSON.parse(raw);
- expect(validate(body), validate.errorText()).toBe(true);
- expect(body.code).toBe("write_only_archive");
- expect(body.surface).toBe("hosted_api");
- // The whole point of the denial is to route the caller somewhere useful.
- expect(body.gateway_url).toBeTruthy();
- });
+ it.each(contentRoutes)(
+ "%s %s refuses with a typed 409",
+ async (method, path, query) => {
+ const { request } = await import("./targets/hosted-inproc");
+ const res = await request({
+ method,
+ path,
+ query,
+ headers: authHeader(),
+ });
+ const raw = await res.clone().text();
+ expect(
+ res.status,
+ `${path} -> ${res.status}: ${raw.slice(0, 300)}`,
+ ).toBe(409);
+ const validate = validatorFor(method, specPathFor(path), 409)!;
+ const body = JSON.parse(raw);
+ expect(validate(body), validate.errorText()).toBe(true);
+ expect(body.code).toBe("write_only_archive");
+ expect(body.surface).toBe("hosted_api");
+ // The whole point of the denial is to route the caller somewhere useful.
+ expect(body.gateway_url).toBeTruthy();
+ },
+ );
- it("does NOT refuse /devices — fleet metadata is control-plane, not content", async () => {
- const { request } = await import("./targets/hosted-inproc");
- const res = await request({ path: "/api/enterprise/v1/devices", headers: authHeader() });
- expect(res.status).toBe(200);
- });
-});
+ it("does NOT refuse /devices — fleet metadata is control-plane, not content", async () => {
+ const { request } = await import("./targets/hosted-inproc");
+ const res = await request({
+ path: "/api/enterprise/v1/devices",
+ headers: authHeader(),
+ });
+ expect(res.status).toBe(200);
+ });
+ },
+);
diff --git a/crates/screenpipe-gateway/e2e/conformance/strictness.test.ts b/crates/screenpipe-gateway/e2e/conformance/strictness.test.ts
index eb3ffba07..a97badb0b 100644
--- a/crates/screenpipe-gateway/e2e/conformance/strictness.test.ts
+++ b/crates/screenpipe-gateway/e2e/conformance/strictness.test.ts
@@ -61,6 +61,7 @@ const goodDeviceList = () => ({
{
device_id: "dev-alice",
label: "alice-mbp",
+ member_email: "[email protected]",
platform: "macos",
app_version: "2.5.136",
last_seen: "2026-07-22T09:08:00+00:00",
@@ -69,6 +70,7 @@ const goodDeviceList = () => ({
{
device_id: "dev-bob",
label: "bob-thinkpad",
+ member_email: null,
platform: null,
app_version: null,
last_seen: "2026-07-22T10:08:00Z",
@@ -125,6 +127,16 @@ describe("the good bodies validate (or every rejection below proves nothing)", (
const v = componentValidator(name as string);
expect(v(body), v.errorText()).toBe(true);
});
+
+ it.each([
+ ["DeviceList", goodDeviceList()],
+ ["SearchResult", goodSearchResult()],
+ ["RollupList", goodRollupList()],
+ ])("%s also accepts the opt-in pagination field", (name, body: any) => {
+ body.next_cursor = null;
+ const v = componentValidator(name);
+ expect(v(body), v.errorText()).toBe(true);
+ });
});
describe("renaming a required field is REJECTED", () => {
@@ -167,7 +179,7 @@ describe("renaming a required field is REJECTED", () => {
body.files[0][`x_${field}`] = body.files[0][field];
delete body.files[0][field];
expect(componentValidator("FileList")(body)).toBe(false);
- }
+ },
);
});
@@ -263,7 +275,9 @@ describe("nulling a non-nullable field is REJECTED", () => {
describe("the nullable fields ARE allowed to be null (accepted divergences)", () => {
it("every RecordSummary field except kind", () => {
- const nullable = Object.keys(goodRecordSummary()).filter((k) => k !== "kind");
+ const nullable = Object.keys(goodRecordSummary()).filter(
+ (k) => k !== "kind",
+ );
for (const field of nullable) {
const body: any = goodRecordSummary();
body[field] = null;
@@ -279,6 +293,12 @@ describe("the nullable fields ARE allowed to be null (accepted divergences)", ()
expect(componentValidator("DeviceList")(body)).toBe(true);
});
+ it("Device.member_email — null when no verified identity source exists (DEV-MEMBER-EMAIL)", () => {
+ const body: any = goodDeviceList();
+ body.devices[0].member_email = null;
+ expect(componentValidator("DeviceList")(body)).toBe(true);
+ });
+
it("FileEntry.size / last_modified — nullable on the gateway (FILES-NULLABILITY)", () => {
const body: any = goodFileList();
body.files[0].size = null;
@@ -288,10 +308,22 @@ describe("the nullable fields ARE allowed to be null (accepted divergences)", ()
});
describe("removing a required field is REJECTED", () => {
+ it("Device.member_email", () => {
+ const body: any = goodDeviceList();
+ delete body.devices[0].member_email;
+ expect(componentValidator("DeviceList")(body)).toBe(false);
+ });
+
it.each([
["DeviceList", ["count", "devices"]],
- ["SearchResult", ["query", "result_count", "results", "truncated", "window_hours"]],
- ["FileList", ["device_id", "count", "files", "next_cursor", "window_hours"]],
+ [
+ "SearchResult",
+ ["query", "result_count", "results", "truncated", "window_hours"],
+ ],
+ [
+ "FileList",
+ ["device_id", "count", "files", "next_cursor", "window_hours"],
+ ],
["RollupList", ["ok", "device", "from", "to", "count", "rollups"]],
])("%s", (name, fields) => {
for (const field of fields as string[]) {
@@ -304,7 +336,7 @@ describe("removing a required field is REJECTED", () => {
delete body[field];
expect(
componentValidator(name as string)(body),
- `${name} validated without required '${field}'`
+ `${name} validated without required '${field}'`,
).toBe(false);
}
});
@@ -313,7 +345,9 @@ describe("removing a required field is REJECTED", () => {
describe("error bodies are strict too", () => {
it("{error} accepts only that", () => {
expect(componentValidator("Error")({ error: "not found" })).toBe(true);
- expect(componentValidator("Error")({ error: "x", stack: "..." })).toBe(false);
+ expect(componentValidator("Error")({ error: "x", stack: "..." })).toBe(
+ false,
+ );
expect(componentValidator("Error")({})).toBe(false);
});
@@ -333,17 +367,23 @@ describe("error bodies are strict too", () => {
it("the write-only denial code and surface are pinned", () => {
const v = componentValidator("WriteOnlyDenial");
- expect(v({ error: "x", code: "write_only_archive", surface: "hosted_api" })).toBe(true);
+ expect(
+ v({ error: "x", code: "write_only_archive", surface: "hosted_api" }),
+ ).toBe(true);
expect(
v({
error: "x",
code: "write_only_archive",
surface: "hosted_api",
gateway_url: "https://gw",
- })
+ }),
).toBe(true);
- expect(v({ error: "x", code: "write_only", surface: "hosted_api" })).toBe(false);
- expect(v({ error: "x", code: "write_only_archive", surface: "made_up" })).toBe(false);
+ expect(v({ error: "x", code: "write_only", surface: "hosted_api" })).toBe(
+ false,
+ );
+ expect(
+ v({ error: "x", code: "write_only_archive", surface: "made_up" }),
+ ).toBe(false);
});
});
@@ -351,17 +391,25 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {
it("every JSON response object schema sets additionalProperties: false", () => {
// The one deliberate exception is documented in the spec: a rollup's `data`
// and an artifact body are passthroughs whose shape the route does not own.
- const allowed = new Set(["Rollup.data", "GeneratedWorkflows.artifacts.items"]);
+ const allowed = new Set([
+ "Rollup.data",
+ "GeneratedWorkflows.artifacts.items",
+ ]);
const offenders: string[] = [];
walk(rawSpec().components.schemas, "");
function walk(node: any, path: string) {
if (!node || typeof node !== "object") return;
- if (Array.isArray(node)) return node.forEach((n, i) => walk(n, `${path}[${i}]`));
+ if (Array.isArray(node))
+ return node.forEach((n, i) => walk(n, `${path}[${i}]`));
const isObjectSchema =
node.type === "object" ||
(Array.isArray(node.type) && node.type.includes("object")) ||
!!node.properties;
- if (isObjectSchema && node.additionalProperties !== false && !allowed.has(path)) {
+ if (
+ isObjectSchema &&
+ node.additionalProperties !== false &&
+ !allowed.has(path)
+ ) {
offenders.push(path || "<root>");
}
for (const [k, v] of Object.entries(node)) {
@@ -374,7 +422,10 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {
}
}
}
- expect(offenders, `these object schemas would validate a renamed field`).toEqual([]);
+ expect(
+ offenders,
+ `these object schemas would validate a renamed field`,
+ ).toEqual([]);
});
it("every RESPONSE object schema has an exhaustive required list", () => {
@@ -402,10 +453,17 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {
}
expect(
requestSchemas.size,
- "no request schemas found — the derivation broke and this test is now vacuous"
+ "no request schemas found — the derivation broke and this test is now vacuous",
).toBeGreaterThan(0);
- const optionalByDesign = new Set(["WriteOnlyDenial.gateway_url"]);
+ const optionalByDesign = new Set([
+ "WriteOnlyDenial.gateway_url",
+ // Pagination is opt-in so legacy responses retain their exact shape.
+ "DeviceList.next_cursor",
+ "SearchResult.next_cursor",
+ "RecordList.next_cursor",
+ "RollupList.next_cursor",
+ ]);
const gaps: string[] = [];
for (const [name, schema] of Object.entries<any>(doc.components.schemas)) {
if (!schema.properties) continue;
@@ -418,7 +476,10 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {
}
}
}
- expect(gaps, "these properties could vanish without the schema noticing").toEqual([]);
+ expect(
+ gaps,
+ "these properties could vanish without the schema noticing",
+ ).toEqual([]);
});
it("every operation declares an x-scope, or is an explicitly public route", () => {
@@ -426,7 +487,7 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {
expect(
op.scope !== null || op.isPublic,
`${op.method} ${op.path} has neither x-scope nor x-public: true — the ` +
- `auth matrix cannot classify it, so it would be silently unasserted`
+ `auth matrix cannot classify it, so it would be silently unasserted`,
).toBe(true);
}
});
@@ -437,7 +498,7 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {
for (const id of op.divergences) {
expect(
Object.keys(ledger),
- `${op.method} ${op.path} claims divergence '${id}'`
+ `${op.method} ${op.path} claims divergence '${id}'`,
).toContain(id);
}
}
@@ -447,19 +508,22 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {
for (const [id, entry] of Object.entries<any>(divergenceLedger())) {
expect(entry.rationale, `${id} has no rationale`).toBeTruthy();
expect(entry.hosted, `${id} does not say what hosted does`).toBeTruthy();
- expect(entry.gateway, `${id} does not say what the gateway does`).toBeTruthy();
+ expect(
+ entry.gateway,
+ `${id} does not say what the gateway does`,
+ ).toBeTruthy();
}
});
it("validatorFor THROWS for a status the spec does not describe", () => {
- expect(() => validatorFor("GET", "/api/enterprise/v1/devices", 418)).toThrow(
- /does not define a 418 response/
- );
+ expect(() =>
+ validatorFor("GET", "/api/enterprise/v1/devices", 418),
+ ).toThrow(/does not define a 418 response/);
});
it("validatorFor THROWS for an operation the spec does not describe", () => {
- expect(() => validatorFor("GET", "/api/enterprise/v1/experimental", 200)).toThrow(
- /does not describe/
- );
+ expect(() =>
+ validatorFor("GET", "/api/enterprise/v1/experimental", 200),
+ ).toThrow(/does not describe/);
});
});
diff --git a/crates/screenpipe-gateway/src/api.rs b/crates/screenpipe-gateway/src/api.rs
index 32093cc4c..f83f8a3ed 100644
--- a/crates/screenpipe-gateway/src/api.rs
+++ b/crates/screenpipe-gateway/src/api.rs
@@ -60,10 +60,12 @@ use axum::{
routing::{any, get, MethodRouter},
Json, Router,
};
+use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};
use chrono::{DateTime, Duration as ChronoDuration, SecondsFormat, Utc};
use screenpipe_db::DatabaseManager;
use screenpipe_sync::{BlobSource, ListRequest};
use screenpipe_telemetry_wire::{frame_image_key, org_telemetry_prefix, sanitize_id};
+use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use tracing::warn;
@@ -411,6 +413,84 @@ fn parse_limit(params: &HashMap<String, String>) -> Result<i64, Box<Response>> {
parse_bounded_int(params, "limit", DEFAULT_LIMIT, 1, MAX_LIMIT)
}
+#[derive(Serialize, Deserialize)]
+struct CursorEnvelope {
+ v: u8,
+ endpoint: String,
+ filters: String,
+ state: Value,
+}
+
+fn invalid_cursor() -> Box<Response> {
+ err_json_code(
+ StatusCode::BAD_REQUEST,
+ INVALID_QUERY_PARAM,
+ "invalid 'cursor': expected a cursor returned by this endpoint with the same filters",
+ )
+}
+
+fn encode_cursor(endpoint: &str, filters: &str, state: Value) -> String {
+ URL_SAFE_NO_PAD.encode(
+ serde_json::to_vec(&CursorEnvelope {
+ v: 1,
+ endpoint: endpoint.to_string(),
+ filters: filters.to_string(),
+ state,
+ })
+ .expect("cursor envelope serializes"),
+ )
+}
+
+fn parse_cursor(
+ params: &HashMap<String, String>,
+ endpoint: &str,
+ filters: &str,
+) -> Result<Option<Value>, Box<Response>> {
+ let Some(raw) = param(params, "cursor") else {
+ return Ok(None);
+ };
+ if raw.len() > 4096 {
+ return Err(invalid_cursor());
+ }
+ let envelope: CursorEnvelope = URL_SAFE_NO_PAD
+ .decode(raw)
+ .ok()
+ .and_then(|bytes| serde_json::from_slice(&bytes).ok())
+ .filter(|c: &CursorEnvelope| {
+ c.v == 1 && c.endpoint == endpoint && c.filters == filters && c.state.is_object()
+ })
+ .ok_or_else(invalid_cursor)?;
+ Ok(Some(envelope.state))
+}
+
+fn cursor_string(state: &Value, name: &str) -> Result<String, Box<Response>> {
+ state
+ .get(name)
+ .and_then(Value::as_str)
+ .map(str::to_string)
+ .ok_or_else(invalid_cursor)
+}
+
+fn filter_signature(values: Value) -> String {
+ serde_json::to_string(&values).expect("filter signature serializes")
+}
+
+fn cursor_window(state: &Value) -> Result<TimeWindow, Box<Response>> {
+ let since = cursor_string(state, "since")?;
+ let until = cursor_string(state, "until")?;
+ let since = DateTime::parse_from_rfc3339(&since)
+ .map(|t| t.with_timezone(&Utc))
+ .map_err(|_| invalid_cursor())?;
+ let until = DateTime::parse_from_rfc3339(&until)
+ .map(|t| t.with_timezone(&Utc))
+ .map_err(|_| invalid_cursor())?;
+ Ok(TimeWindow {
+ since,
+ until,
+ window_hours: ((until - since).num_milliseconds() as f64 / 3_600_000.0).max(0.0),
+ })
+}
+
fn rfc3339z(t: &DateTime<Utc>) -> String {
t.to_rfc3339_opts(SecondsFormat::Millis, true)
}
@@ -492,13 +572,40 @@ struct KindQuery<'a> {
until: String,
limit: i64,
newest_first: bool,
+ cursor_t: Option<&'a str>,
+ cursor_id: Option<&'a str>,
+}
+
+fn with_cursor_id(mut record: Value, id: String) -> Value {
+ record["_cursor_id"] = Value::String(id);
+ record
+}
+
+fn cursor_id_of(v: &Value) -> &str {
+ v.get("_cursor_id").and_then(Value::as_str).unwrap_or("")
+}
+
+fn compare_records(a: &Value, b: &Value, newest_first: bool) -> std::cmp::Ordering {
+ let by_time = match (
+ DateTime::parse_from_rfc3339(t_of(a)),
+ DateTime::parse_from_rfc3339(t_of(b)),
+ ) {
+ (Ok(a), Ok(b)) => a.cmp(&b),
+ _ => t_of(a).cmp(t_of(b)),
+ };
+ let order = by_time.then_with(|| cursor_id_of(a).cmp(cursor_id_of(b)));
+ if newest_first {
+ order.reverse()
+ } else {
+ order
+ }
}
async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>, sqlx::Error> {
let fts = kq.q.and_then(fts_expression);
let order = if kq.newest_first { "DESC" } else { "ASC" };
let sql = format!(
- r#"SELECT f.sync_id, f.machine_id, f.device_name, f.timestamp, f.app_name,
+ r#"SELECT f.id, f.sync_id, f.machine_id, f.device_name, f.timestamp, f.app_name,
f.window_name, f.browser_url, f.full_text
FROM frames f
{fts_join}
@@ -506,10 +613,13 @@ async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va
AND datetime(f.timestamp) >= datetime(?1) AND datetime(f.timestamp) <= datetime(?2)
AND (?3 IS NULL OR f.machine_id = ?3)
AND (?4 IS NULL OR f.app_name IS NULL OR lower(f.app_name) = lower(?4))
- ORDER BY datetime(f.timestamp) {order}
- LIMIT ?5"#,
+ AND (?5 IS NULL OR datetime(f.timestamp) {cmp} datetime(?5)
+ OR (datetime(f.timestamp) = datetime(?5) AND printf('frame:%020d', f.id) {cmp} ?6))
+ ORDER BY datetime(f.timestamp) {order}, printf('frame:%020d', f.id) {order}
+ LIMIT ?7"#,
+ cmp = if kq.newest_first { "<" } else { ">" },
fts_join = if fts.is_some() {
- "JOIN frames_fts ON frames_fts.rowid = f.id AND frames_fts MATCH ?6"
+ "JOIN frames_fts ON frames_fts.rowid = f.id AND frames_fts MATCH ?8"
} else {
""
},
@@ -517,6 +627,7 @@ async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va
let mut query = sqlx::query_as::<
_,
(
+ i64,
Option<String>,
Option<String>,
Option<String>,
@@ -531,6 +642,8 @@ async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va
.bind(&kq.until)
.bind(kq.device_id)
.bind(kq.app_name)
+ .bind(kq.cursor_t)
+ .bind(kq.cursor_id)
.bind(kq.limit);
if let Some(f) = &fts {
query = query.bind(f.clone());
@@ -539,24 +652,27 @@ async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va
Ok(rows
.into_iter()
.map(
- |(sync_id, machine_id, device_name, ts, app, window, url, text)| {
- record_summary(
- "frame",
- Some(ts),
- device_name,
- machine_id,
- app,
- window,
- url,
- text,
- None,
- None,
- None,
- None,
- None,
- None,
- local_id_from_sync_id(sync_id.as_deref()),
- None,
+ |(id, sync_id, machine_id, device_name, ts, app, window, url, text)| {
+ with_cursor_id(
+ record_summary(
+ "frame",
+ Some(ts),
+ device_name,
+ machine_id,
+ app,
+ window,
+ url,
+ text,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ local_id_from_sync_id(sync_id.as_deref()),
+ None,
+ ),
+ format!("frame:{id:020}"),
)
},
)
@@ -567,17 +683,20 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va
let fts = kq.q.and_then(fts_expression);
let order = if kq.newest_first { "DESC" } else { "ASC" };
let sql = format!(
- r#"SELECT p.device_id, p.device_label, p.timestamp, p.app_name,
+ r#"SELECT p.rowid, p.device_id, p.device_label, p.timestamp, p.app_name,
p.window_name, p.browser_url, p.text, p.frame_id
FROM gateway_parsed_records p
{fts_join}
WHERE datetime(p.timestamp) >= datetime(?1) AND datetime(p.timestamp) <= datetime(?2)
AND (?3 IS NULL OR p.device_id = ?3)
AND (?4 IS NULL OR lower(p.app_name) = lower(?4))
- ORDER BY datetime(p.timestamp) {order}
- LIMIT ?5"#,
+ AND (?5 IS NULL OR datetime(p.timestamp) {cmp} datetime(?5)
+ OR (datetime(p.timestamp) = datetime(?5) AND printf('parsed:%020d', p.rowid) {cmp} ?6))
+ ORDER BY datetime(p.timestamp) {order}, printf('parsed:%020d', p.rowid) {order}
+ LIMIT ?7"#,
+ cmp = if kq.newest_first { "<" } else { ">" },
fts_join = if fts.is_some() {
- "JOIN gateway_parsed_records_fts fts ON fts.rowid = p.rowid AND gateway_parsed_records_fts MATCH ?6"
+ "JOIN gateway_parsed_records_fts fts ON fts.rowid = p.rowid AND gateway_parsed_records_fts MATCH ?8"
} else {
""
},
@@ -585,6 +704,7 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va
let mut query = sqlx::query_as::<
_,
(
+ i64,
String,
String,
String,
@@ -599,6 +719,8 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va
.bind(&kq.until)
.bind(kq.device_id)
.bind(kq.app_name)
+ .bind(kq.cursor_t)
+ .bind(kq.cursor_id)
.bind(kq.limit);
if let Some(f) = &fts {
query = query.bind(f.clone());
@@ -606,26 +728,31 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va
let rows = query.fetch_all(&db.pool).await?;
Ok(rows
.into_iter()
- .map(|(device_id, label, ts, app, window, url, text, frame_id)| {
- record_summary(
- "parsed",
- Some(ts),
- Some(label),
- Some(device_id),
- Some(app),
- Some(window),
- url,
- Some(text),
- None,
- None,
- None,
- None,
- None,
- None,
- Some(frame_id),
- None,
- )
- })
+ .map(
+ |(id, device_id, label, ts, app, window, url, text, frame_id)| {
+ with_cursor_id(
+ record_summary(
+ "parsed",
+ Some(ts),
+ Some(label),
+ Some(device_id),
+ Some(app),
+ Some(window),
+ url,
+ Some(text),
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ Some(frame_id),
+ None,
+ ),
+ format!("parsed:{id:020}"),
+ )
+ },
+ )
.collect())
}
@@ -633,7 +760,7 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Val
let fts = kq.q.and_then(fts_expression);
let order = if kq.newest_first { "DESC" } else { "ASC" };
let sql = format!(
- r#"SELECT at.sync_id, ac.machine_id, gd.device_label, at.timestamp,
+ r#"SELECT at.id, at.sync_id, ac.machine_id, gd.device_label, at.timestamp,
at.transcription, s.name
FROM audio_transcriptions at
JOIN audio_chunks ac ON at.audio_chunk_id = ac.id
@@ -643,10 +770,13 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Val
WHERE ac.machine_id IS NOT NULL
AND datetime(at.timestamp) >= datetime(?1) AND datetime(at.timestamp) <= datetime(?2)
AND (?3 IS NULL OR ac.machine_id = ?3)
- ORDER BY datetime(at.timestamp) {order}
- LIMIT ?4"#,
+ AND (?4 IS NULL OR datetime(at.timestamp) {cmp} datetime(?4)
+ OR (datetime(at.timestamp) = datetime(?4) AND printf('audio:%020d', at.id) {cmp} ?5))
+ ORDER BY datetime(at.timestamp) {order}, printf('audio:%020d', at.id) {order}
+ LIMIT ?6"#,
+ cmp = if kq.newest_first { "<" } else { ">" },
fts_join = if fts.is_some() {
- "JOIN audio_transcriptions_fts fts ON fts.rowid = at.id AND audio_transcriptions_fts MATCH ?5"
+ "JOIN audio_transcriptions_fts fts ON fts.rowid = at.id AND audio_transcriptions_fts MATCH ?7"
} else {
""
},
@@ -654,6 +784,7 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Val
let mut query = sqlx::query_as::<
_,
(
+ i64,
Option<String>,
Option<String>,
Option<String>,
@@ -665,6 +796,8 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Val
.bind(&kq.since)
.bind(&kq.until)
.bind(kq.device_id)
+ .bind(kq.cursor_t)
+ .bind(kq.cursor_id)
.bind(kq.limit);
if let Some(f) = &fts {
query = query.bind(f.clone());
@@ -673,24 +806,27 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Val
Ok(rows
.into_iter()
.map(
- |(_sync_id, machine_id, label, ts, transcription, speaker)| {
- record_summary(
- "audio",
- Some(ts),
- label,
- machine_id,
- None,
- None,
- None,
- None,
- Some(transcription),
- speaker,
- None,
- None,
- None,
- None,
- None,
- None,
+ |(id, _sync_id, machine_id, label, ts, transcription, speaker)| {
+ with_cursor_id(
+ record_summary(
+ "audio",
+ Some(ts),
+ label,
+ machine_id,
+ None,
+ None,
+ None,
+ None,
+ Some(transcription),
+ speaker,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ ),
+ format!("audio:{id:020}"),
)
},
)
@@ -701,7 +837,7 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>
let fts = kq.q.and_then(fts_expression);
let order = if kq.newest_first { "DESC" } else { "ASC" };
let sql = format!(
- r#"SELECT ue.sync_id, ue.machine_id, gd.device_label, ue.timestamp,
+ r#"SELECT ue.id, ue.sync_id, ue.machine_id, gd.device_label, ue.timestamp,
ue.app_name, ue.browser_url
FROM ui_events ue
LEFT JOIN gateway_devices gd ON gd.device_id = ue.machine_id
@@ -710,10 +846,13 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>
AND datetime(ue.timestamp) >= datetime(?1) AND datetime(ue.timestamp) <= datetime(?2)
AND (?3 IS NULL OR ue.machine_id = ?3)
AND (?4 IS NULL OR ue.app_name IS NULL OR lower(ue.app_name) = lower(?4))
- ORDER BY datetime(ue.timestamp) {order}
- LIMIT ?5"#,
+ AND (?5 IS NULL OR datetime(ue.timestamp) {cmp} datetime(?5)
+ OR (datetime(ue.timestamp) = datetime(?5) AND printf('ui:%020d', ue.id) {cmp} ?6))
+ ORDER BY datetime(ue.timestamp) {order}, printf('ui:%020d', ue.id) {order}
+ LIMIT ?7"#,
+ cmp = if kq.newest_first { "<" } else { ">" },
fts_join = if fts.is_some() {
- "JOIN ui_events_fts ON ui_events_fts.rowid = ue.id AND ui_events_fts MATCH ?6"
+ "JOIN ui_events_fts ON ui_events_fts.rowid = ue.id AND ui_events_fts MATCH ?8"
} else {
""
},
@@ -721,6 +860,7 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>
let mut query = sqlx::query_as::<
_,
(
+ i64,
Option<String>,
Option<String>,
Option<String>,
@@ -733,6 +873,8 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>
.bind(&kq.until)
.bind(kq.device_id)
.bind(kq.app_name)
+ .bind(kq.cursor_t)
+ .bind(kq.cursor_id)
.bind(kq.limit);
if let Some(f) = &fts {
query = query.bind(f.clone());
@@ -743,24 +885,27 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>
// on the hosted side too.
Ok(rows
.into_iter()
- .map(|(_sync_id, machine_id, label, ts, app, url)| {
- record_summary(
- "ui",
- Some(ts),
- label,
- machine_id,
- app,
- None,
- url,
- None,
- None,
- None,
- None,
- None,
- None,
- None,
- None,
- None,
+ .map(|(id, _sync_id, machine_id, label, ts, app, url)| {
+ with_cursor_id(
+ record_summary(
+ "ui",
+ Some(ts),
+ label,
+ machine_id,
+ app,
+ None,
+ url,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ ),
+ format!("ui:{id:020}"),
)
})
.collect())
@@ -773,7 +918,7 @@ async fn query_memories(
let fts = kq.q.and_then(fts_expression);
let order = if kq.newest_first { "DESC" } else { "ASC" };
let sql = format!(
- r#"SELECT m.sync_uuid, m.sync_modified_by, gd.device_label, m.created_at,
+ r#"SELECT m.id, m.sync_uuid, m.sync_modified_by, gd.device_label, m.created_at,
m.content, m.importance, m.tags, m.source
FROM memories m
LEFT JOIN gateway_devices gd ON gd.device_id = m.sync_modified_by
@@ -781,10 +926,13 @@ async fn query_memories(
WHERE m.sync_modified_by IS NOT NULL
AND datetime(m.created_at) >= datetime(?1) AND datetime(m.created_at) <= datetime(?2)
AND (?3 IS NULL OR m.sync_modified_by = ?3)
- ORDER BY datetime(m.created_at) {order}
- LIMIT ?4"#,
+ AND (?4 IS NULL OR datetime(m.created_at) {cmp} datetime(?4)
+ OR (datetime(m.created_at) = datetime(?4) AND printf('memory:%020d', m.id) {cmp} ?5))
+ ORDER BY datetime(m.created_at) {order}, printf('memory:%020d', m.id) {order}
+ LIMIT ?6"#,
+ cmp = if kq.newest_first { "<" } else { ">" },
fts_join = if fts.is_some() {
- "JOIN memories_fts ON memories_fts.rowid = m.id AND memories_fts MATCH ?5"
+ "JOIN memories_fts ON memories_fts.rowid = m.id AND memories_fts MATCH ?7"
} else {
""
},
@@ -792,6 +940,7 @@ async fn query_memories(
let mut query = sqlx::query_as::<
_,
(
+ i64,
Option<String>,
Option<String>,
Option<String>,
@@ -805,6 +954,8 @@ async fn query_memories(
.bind(&kq.since)
.bind(&kq.until)
.bind(kq.device_id)
+ .bind(kq.cursor_t)
+ .bind(kq.cursor_id)
.bind(kq.limit);
if let Some(f) = &fts {
query = query.bind(f.clone());
@@ -813,25 +964,28 @@ async fn query_memories(
Ok(rows
.into_iter()
.map(
- |(sync_uuid, machine_id, label, created_at, content, importance, tags, source)| {
+ |(id, sync_uuid, machine_id, label, created_at, content, importance, tags, source)| {
let tags: Option<Vec<String>> = tags.and_then(|t| serde_json::from_str(&t).ok());
- record_summary(
- "memory",
- Some(created_at),
- label,
- machine_id,
- None,
- None,
- None,
- None,
- None,
- None,
- Some(content),
- importance,
- tags,
- source,
- None,
- local_id_from_sync_id(sync_uuid.as_deref()),
+ with_cursor_id(
+ record_summary(
+ "memory",
+ Some(created_at),
+ label,
+ machine_id,
+ None,
+ None,
+ None,
+ None,
+ None,
+ None,
+ Some(content),
+ importance,
+ tags,
+ source,
+ None,
+ local_id_from_sync_id(sync_uuid.as_deref()),
+ ),
+ format!("memory:{id:020}"),
)
},
)
@@ -858,8 +1012,11 @@ async fn query_feedback(
OR lower(COALESCE(f.comment, '')) LIKE '%' || lower(?4) || '%'
OR lower(COALESCE(f.snapshot, '')) LIKE '%' || lower(?4) || '%'
OR lower(f.context) LIKE '%' || lower(?4) || '%')
- ORDER BY datetime(f.updated_at) {order}
- LIMIT ?5"#,
+ AND (?5 IS NULL OR datetime(f.updated_at) {cmp} datetime(?5)
+ OR (datetime(f.updated_at) = datetime(?5) AND 'feedback:' || f.id {cmp} ?6))
+ ORDER BY datetime(f.updated_at) {order}, 'feedback:' || f.id {order}
+ LIMIT ?7"#,
+ cmp = if kq.newest_first { "<" } else { ">" },
);
let rows = sqlx::query_as::<
_,
@@ -884,6 +1041,8 @@ async fn query_feedback(
.bind(&kq.until)
.bind(kq.device_id)
.bind(kq.q)
+ .bind(kq.cursor_t)
+ .bind(kq.cursor_id)
.bind(kq.limit)
.fetch_all(&db.pool)
.await?;
@@ -906,7 +1065,7 @@ async fn query_feedback(
context,
created_at,
)| {
- json!({
+ with_cursor_id(json!({
"kind": "feedback",
"t": updated_at,
"device": device,
@@ -938,7 +1097,7 @@ async fn query_feedback(
"context": serde_json::from_str::<Value>(&context).unwrap_or_else(|_| json!({})),
"created_at": created_at,
"updated_at": updated_at,
- })
+ }), format!("feedback:{id}"))
},
)
.collect())
@@ -972,26 +1131,90 @@ fn t_of(v: &Value) -> &str {
// ─── GET /api/enterprise/v1/devices ─────────────────────────────────────────
-async fn devices(State(state): State<ApiState>) -> Response {
- let rows: Result<Vec<(String, String, String, String)>, sqlx::Error> = sqlx::query_as(
- "SELECT device_id, device_label, enrolled_at, last_seen \
- FROM gateway_devices ORDER BY last_seen DESC",
- )
- .fetch_all(&state.db.pool)
- .await;
+async fn devices(
+ State(state): State<ApiState>,
+ Query(params): Query<HashMap<String, String>>,
+) -> Response {
+ let keyset = param(&params, "pagination") == Some("keyset");
+ let limit = if keyset {
+ match parse_limit(&params) {
+ Ok(v) => v as usize,
+ Err(resp) => return *resp,
+ }
+ } else {
+ usize::MAX
+ };
+ let filters = "{}";
+ let cursor = if keyset {
+ match parse_cursor(&params, "devices", filters) {
+ Ok(v) => v,
+ Err(resp) => return *resp,
+ }
+ } else {
+ None
+ };
+ let (as_of, after_device_id) = match cursor {
+ Some(ref state) => match (
+ cursor_string(state, "as_of"),
+ cursor_string(state, "device_id"),
+ ) {
+ (Ok(as_of), Ok(device_id)) => (as_of, Some(device_id)),
+ _ => return *invalid_cursor(),
+ },
+ None => (rfc3339z(&Utc::now()), None),
+ };
+ let rows: Result<Vec<(String, String, String, String)>, sqlx::Error> = if keyset {
+ sqlx::query_as(
+ "SELECT device_id, device_label, enrolled_at, last_seen \
+ FROM gateway_devices \
+ WHERE datetime(enrolled_at) <= datetime(?1) \
+ AND (?2 IS NULL OR device_id > ?2) \
+ ORDER BY device_id ASC LIMIT ?3",
+ )
+ .bind(&as_of)
+ .bind(after_device_id.as_deref())
+ .bind((limit + 1) as i64)
+ .fetch_all(&state.db.pool)
+ .await
+ } else {
+ sqlx::query_as(
+ "SELECT device_id, device_label, enrolled_at, last_seen \
+ FROM gateway_devices ORDER BY last_seen DESC",
+ )
+ .fetch_all(&state.db.pool)
+ .await
+ };
match rows {
- Ok(rows) => Json(json!({
- "count": rows.len(),
- "devices": rows.into_iter().map(|(device_id, label, enrolled_at, last_seen)| json!({
- "device_id": device_id,
- "label": label,
- "platform": null,
- "app_version": null,
- "last_seen": last_seen,
- "enrolled_at": enrolled_at,
- })).collect::<Vec<_>>(),
- }))
- .into_response(),
+ Ok(mut rows) => {
+ let has_more = keyset && rows.len() > limit;
+ if keyset {
+ rows.truncate(limit);
+ }
+ let next_cursor = has_more && !rows.is_empty();
+ let next_cursor = next_cursor.then(|| {
+ encode_cursor(
+ "devices",
+ filters,
+ json!({ "as_of": as_of, "device_id": rows.last().unwrap().0 }),
+ )
+ });
+ let mut body = json!({
+ "count": rows.len(),
+ "devices": rows.into_iter().map(|(device_id, label, enrolled_at, last_seen)| json!({
+ "device_id": device_id,
+ "label": label,
+ "member_email": null,
+ "platform": null,
+ "app_version": null,
+ "last_seen": last_seen,
+ "enrolled_at": enrolled_at,
+ })).collect::<Vec<_>>(),
+ });
+ if keyset {
+ body["next_cursor"] = json!(next_cursor);
+ }
+ Json(body).into_response()
+ }
Err(e) => err_json(
StatusCode::INTERNAL_SERVER_ERROR,
&format!("device list failed: {e}"),
@@ -1005,26 +1228,57 @@ async fn search(
State(state): State<ApiState>,
Query(params): Query<HashMap<String, String>>,
) -> Response {
+ let keyset = param(&params, "pagination") == Some("keyset");
let q = params.get("q").map(|s| s.trim()).unwrap_or("");
let device_id = param(&params, "device_id");
let app_name = param(&params, "app_name");
- let window = match parse_time_window(&params) {
- Ok(w) => w,
- Err(resp) => return *resp,
+ let filters = filter_signature(json!({
+ "q": q,
+ "device_id": device_id,
+ "app_name": app_name,
+ "since": param(&params, "since"),
+ "until": param(&params, "until"),
+ "since_hours_ago": param(&params, "since_hours_ago"),
+ }));
+ let cursor = if keyset {
+ match parse_cursor(&params, "search", &filters) {
+ Ok(v) => v,
+ Err(resp) => return *resp,
+ }
+ } else {
+ None
+ };
+ let window = match cursor.as_ref() {
+ Some(state) => match cursor_window(state) {
+ Ok(w) => w,
+ Err(resp) => return *resp,
+ },
+ None => match parse_time_window(&params) {
+ Ok(w) => w,
+ Err(resp) => return *resp,
+ },
+ };
+ let cursor_position = match cursor.as_ref() {
+ Some(state) => match (cursor_string(state, "t"), cursor_string(state, "id")) {
+ (Ok(t), Ok(id)) => Some((t, id)),
+ _ => return *invalid_cursor(),
+ },
+ None => None,
};
let limit = match parse_limit(&params) {
Ok(l) => l,
Err(resp) => return *resp,
};
-
let kq = KindQuery {
q: if q.is_empty() { None } else { Some(q) },
device_id,
app_name,
since: rfc3339z(&window.since),
until: rfc3339z(&window.until),
- limit,
+ limit: if keyset { limit + 1 } else { limit },
newest_first: true,
+ cursor_t: cursor_position.as_ref().map(|p| p.0.as_str()),
+ cursor_id: cursor_position.as_ref().map(|p| p.1.as_str()),
};
let mut results = match query_kinds(
&state.db,
@@ -1037,10 +1291,33 @@ async fn search(
Err(e) => return err_json(StatusCode::INTERNAL_SERVER_ERROR, &format!("search: {e}")),
};
let records_scanned = results.len();
- results.sort_by(|a, b| t_of(b).cmp(t_of(a)));
+ results.sort_by(|a, b| {
+ if keyset {
+ compare_records(a, b, true)
+ } else {
+ t_of(b).cmp(t_of(a))
+ }
+ });
+ let has_more = keyset && results.len() > limit as usize;
results.truncate(limit as usize);
+ let next_cursor = (has_more && !results.is_empty()).then(|| {
+ let last = results.last().unwrap();
+ encode_cursor(
+ "search",
+ &filters,
+ json!({
+ "since": rfc3339z(&window.since),
+ "until": rfc3339z(&window.until),
+ "t": t_of(last),
+ "id": cursor_id_of(last),
+ }),
+ )
+ });
+ for result in &mut results {
+ result.as_object_mut().map(|o| o.remove("_cursor_id"));
+ }
- Json(json!({
+ let mut body = json!({
"query": q,
"device_id": device_id,
"app_name": app_name,
@@ -1051,8 +1328,11 @@ async fn search(
"truncated": false,
"result_count": results.len(),
"results": results,
- }))
- .into_response()
+ });
+ if keyset {
+ body["next_cursor"] = json!(next_cursor);
+ }
+ Json(body).into_response()
}
// ─── GET /api/enterprise/v1/records ─────────────────────────────────────────
@@ -1061,19 +1341,47 @@ async fn records(
State(state): State<ApiState>,
Query(params): Query<HashMap<String, String>>,
) -> Response {
+ let keyset = param(&params, "pagination") == Some("keyset");
let device_id = param(&params, "device_id");
let kind_filter = param(&params, "kind")
.map(|s| s.to_ascii_lowercase())
.unwrap_or_else(|| "all".to_string());
- let window = match parse_time_window(&params) {
- Ok(w) => w,
- Err(resp) => return *resp,
+ let filters = filter_signature(json!({
+ "device_id": device_id,
+ "kind": kind_filter,
+ "since": param(&params, "since"),
+ "until": param(&params, "until"),
+ "since_hours_ago": param(&params, "since_hours_ago"),
+ }));
+ let cursor = if keyset {
+ match parse_cursor(&params, "records", &filters) {
+ Ok(v) => v,
+ Err(resp) => return *resp,
+ }
+ } else {
+ None
+ };
+ let window = match cursor.as_ref() {
+ Some(state) => match cursor_window(state) {
+ Ok(w) => w,
+ Err(resp) => return *resp,
+ },
+ None => match parse_time_window(&params) {
+ Ok(w) => w,
+ Err(resp) => return *resp,
+ },
+ };
+ let cursor_position = match cursor.as_ref() {
+ Some(state) => match (cursor_string(state, "t"), cursor_string(state, "id")) {
+ (Ok(t), Ok(id)) => Some((t, id)),
+ _ => return *invalid_cursor(),
+ },
+ None => None,
};
let limit = match parse_limit(&params) {
Ok(l) => l,
Err(resp) => return *resp,
};
-
let kinds: Vec<&str> = match kind_filter.as_str() {
"all" => vec!["frame", "parsed", "audio", "ui", "memory", "feedback"],
k => vec![k],
@@ -1084,18 +1392,42 @@ async fn records(
app_name: None,
since: rfc3339z(&window.since),
until: rfc3339z(&window.until),
- limit,
+ limit: if keyset { limit + 1 } else { limit },
newest_first: false,
+ cursor_t: cursor_position.as_ref().map(|p| p.0.as_str()),
+ cursor_id: cursor_position.as_ref().map(|p| p.1.as_str()),
};
let mut recs = match query_kinds(&state.db, &kinds, &kq).await {
Ok(r) => r,
Err(e) => return err_json(StatusCode::INTERNAL_SERVER_ERROR, &format!("records: {e}")),
};
- // Hosted sorts the collected page ascending by `t` (lexicographic).
- recs.sort_by(|a, b| t_of(a).cmp(t_of(b)));
+ recs.sort_by(|a, b| {
+ if keyset {
+ compare_records(a, b, false)
+ } else {
+ t_of(a).cmp(t_of(b))
+ }
+ });
+ let has_more = keyset && recs.len() > limit as usize;
recs.truncate(limit as usize);
+ let next_cursor = (has_more && !recs.is_empty()).then(|| {
+ let last = recs.last().unwrap();
+ encode_cursor(
+ "records",
+ &filters,
+ json!({
+ "since": rfc3339z(&window.since),
+ "until": rfc3339z(&window.until),
+ "t": t_of(last),
+ "id": cursor_id_of(last),
+ }),
+ )
+ });
+ for record in &mut recs {
+ record.as_object_mut().map(|o| o.remove("_cursor_id"));
+ }
- Json(json!({
+ let mut body = json!({
"device_id": device_id,
"kind": kind_filter,
"window_hours": window.window_hours,
@@ -1104,8 +1436,11 @@ async fn records(
"truncated": false,
"record_count": recs.len(),
"records": recs,
- }))
- .into_response()
+ });
+ if keyset {
+ body["next_cursor"] = json!(next_cursor);
+ }
+ Json(body).into_response()
}
// ─── GET /api/enterprise/v1/files ───────────────────────────────────────────
@@ -1115,22 +1450,42 @@ async fn files(
Query(params): Query<HashMap<String, String>>,
) -> Response {
let device_id = param(&params, "device_id");
- let window = match parse_time_window(&params) {
- Ok(w) => w,
- Err(resp) => return *resp,
+ let filters = filter_signature(json!({
+ "device_id": device_id,
+ "since": param(&params, "since"),
+ "until": param(&params, "until"),
+ "since_hours_ago": param(&params, "since_hours_ago"),
+ }));
+ let legacy_offset = param(&params, "cursor").and_then(|raw| raw.parse::<usize>().ok());
+ let cursor = if legacy_offset.is_none() {
+ match parse_cursor(&params, "files", &filters) {
+ Ok(v) => v,
+ Err(resp) => return *resp,
+ }
+ } else {
+ None
+ };
+ let window = match cursor.as_ref() {
+ Some(state) => match cursor_window(state) {
+ Ok(w) => w,
+ Err(resp) => return *resp,
+ },
+ None => match parse_time_window(&params) {
+ Ok(w) => w,
+ Err(resp) => return *resp,
+ },
+ };
+ let after_key = match cursor.as_ref() {
+ Some(state) => match cursor_string(state, "key") {
+ Ok(key) => Some(key),
+ Err(resp) => return *resp,
+ },
+ None => None,
};
let page_size = match parse_bounded_int(&params, "page_size", 100, 1, 1000) {
Ok(v) => v as usize,
Err(resp) => return *resp,
};
- // `cursor` is an OPAQUE token by contract (the hosted target's is a storage
- // continuation token, not a number), so an unparseable one is deliberately
- // page-one rather than a 400 — a client is not supposed to construct it.
- let offset: usize = params
- .get("cursor")
- .and_then(|s| s.parse().ok())
- .unwrap_or(0);
-
let prefix = match device_id {
Some(d) => format!(
"{}{}/",
@@ -1145,21 +1500,32 @@ async fn files(
};
let since = rfc3339z(&window.since);
let until = rfc3339z(&window.until);
- let in_window: Vec<_> = listed
+ let mut in_window: Vec<_> = listed
.entries
.into_iter()
.filter(|e| {
- e.last_modified
- .as_deref()
- .map(|lm| lm >= since.as_str() && lm <= until.as_str())
- .unwrap_or(true)
+ after_key
+ .as_ref()
+ .is_none_or(|key| e.key.as_str() > key.as_str())
+ && e.last_modified
+ .as_deref()
+ .map(|lm| lm >= since.as_str() && lm <= until.as_str())
+ .unwrap_or(true)
})
.collect();
+ in_window.sort_by(|a, b| a.key.cmp(&b.key));
let total = in_window.len();
+ let has_more = legacy_offset.map_or(total > page_size, |offset| {
+ offset.saturating_add(page_size) < total
+ });
+ if let Some(offset) = legacy_offset {
+ in_window = in_window.into_iter().skip(offset).take(page_size).collect();
+ } else {
+ in_window.truncate(page_size);
+ }
+ let last_key = in_window.last().map(|entry| entry.key.clone());
let page: Vec<Value> = in_window
.into_iter()
- .skip(offset)
- .take(page_size)
.map(|e| {
let dev = e
.key
@@ -1176,8 +1542,19 @@ async fn files(
})
})
.collect();
- let next_cursor = if offset + page.len() < total {
- Some((offset + page.len()).to_string())
+ let next_cursor = if has_more && !page.is_empty() {
+ Some(match legacy_offset {
+ Some(offset) => offset.saturating_add(page.len()).to_string(),
+ None => encode_cursor(
+ "files",
+ &filters,
+ json!({
+ "since": rfc3339z(&window.since),
+ "until": rfc3339z(&window.until),
+ "key": last_key.unwrap(),
+ }),
+ ),
+ })
} else {
None
};
@@ -1288,33 +1665,66 @@ async fn rollups(
State(state): State<ApiState>,
Query(params): Query<HashMap<String, String>>,
) -> Response {
- let device: String = param(&params, "device")
- .map(|s| {
- s.chars()
- .map(|c| {
- if c.is_ascii_alphanumeric() || " _.-".contains(c) {
- c
- } else {
- '_'
- }
- })
- .collect()
- })
- .filter(|s: &String| !s.is_empty())
- .unwrap_or_else(|| "org".to_string());
- let today = Utc::now().format("%Y-%m-%d").to_string();
- let to = param(&params, "to").map(str::to_string).unwrap_or(today);
- let from = param(&params, "from")
- .map(str::to_string)
- .unwrap_or_else(|| {
- (Utc::now() - ChronoDuration::days(31))
- .format("%Y-%m-%d")
- .to_string()
- });
+ let keyset = param(&params, "pagination") == Some("keyset");
let limit = match parse_bounded_int(&params, "limit", 35, 1, 100) {
Ok(v) => v as usize,
Err(resp) => return *resp,
};
+ let filters = filter_signature(json!({
+ "device": param(&params, "device"),
+ "from": param(&params, "from"),
+ "to": param(&params, "to"),
+ }));
+ let cursor = if keyset {
+ match parse_cursor(&params, "rollups", &filters) {
+ Ok(v) => v,
+ Err(resp) => return *resp,
+ }
+ } else {
+ None
+ };
+ let (device, from, to, as_of, after) = match cursor {
+ Some(ref state) => match (
+ cursor_string(state, "device"),
+ cursor_string(state, "from"),
+ cursor_string(state, "to"),
+ cursor_string(state, "as_of"),
+ cursor_string(state, "day"),
+ cursor_string(state, "key"),
+ ) {
+ (Ok(device), Ok(from), Ok(to), Ok(as_of), Ok(day), Ok(key)) => {
+ (device, from, to, as_of, Some((day, key)))
+ }
+ _ => return *invalid_cursor(),
+ },
+ None => {
+ let device: String = param(&params, "device")
+ .map(|s| {
+ s.chars()
+ .map(|c| {
+ if c.is_ascii_alphanumeric() || " _.-".contains(c) {
+ c
+ } else {
+ '_'
+ }
+ })
+ .collect()
+ })
+ .filter(|s: &String| !s.is_empty())
+ .unwrap_or_else(|| "org".to_string());
+ let to = param(&params, "to")
+ .map(str::to_string)
+ .unwrap_or_else(|| Utc::now().format("%Y-%m-%d").to_string());
+ let from = param(&params, "from")
+ .map(str::to_string)
+ .unwrap_or_else(|| {
+ (Utc::now() - ChronoDuration::days(31))
+ .format("%Y-%m-%d")
+ .to_string()
+ });
+ (device, from, to, rfc3339z(&Utc::now()), None)
+ }
+ };
let prefix = format!("rollups/{}/{}/", sanitize_id(&state.license_id, 64), device);
let listed = match state.source.list(&ListRequest::new(&prefix)).await {
@@ -1325,18 +1735,32 @@ async fn rollups(
.entries
.into_iter()
.filter_map(|e| {
+ if keyset
+ && e.last_modified
+ .as_deref()
+ .is_some_and(|modified| modified > as_of.as_str())
+ {
+ return None;
+ }
let day = e.key.strip_prefix(&prefix)?.strip_suffix(".json")?;
let ok = day.len() == 10
&& day.chars().enumerate().all(|(i, c)| match i {
4 | 7 => c == '-',
_ => c.is_ascii_digit(),
});
- (ok && day >= from.as_str() && day <= to.as_str())
+ let after_cursor = after.as_ref().is_none_or(|(after_day, after_key)| {
+ day < after_day.as_str()
+ || (day == after_day && e.key.as_str() < after_key.as_str())
+ });
+ (ok && day >= from.as_str() && day <= to.as_str() && after_cursor)
.then(|| (day.to_string(), e.key.clone()))
})
.collect();
- days.sort_by(|a, b| b.0.cmp(&a.0));
+ days.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| b.1.cmp(&a.1)));
+ let has_more = keyset && days.len() > limit;
days.truncate(limit);
+ let selected_count = days.len();
+ let last_position = days.last().cloned();
let mut out = Vec::with_capacity(days.len());
for (day, key) in days {
@@ -1349,15 +1773,33 @@ async fn rollups(
}
}
- Json(json!({
+ let next_cursor = (has_more && selected_count > 0).then(|| {
+ let (day, key) = last_position.unwrap();
+ encode_cursor(
+ "rollups",
+ &filters,
+ json!({
+ "device": &device,
+ "from": &from,
+ "to": &to,
+ "as_of": &as_of,
+ "day": day,
+ "key": key,
+ }),
+ )
+ });
+ let mut body = json!({
"ok": true,
"device": device,
"from": from,
"to": to,
"count": out.len(),
"rollups": out,
- }))
- .into_response()
+ });
+ if keyset {
+ body["next_cursor"] = json!(next_cursor);
+ }
+ Json(body).into_response()
}
#[cfg(test)]
@@ -1535,6 +1977,22 @@ mod tests {
}
}
+ #[test]
+ fn keyset_cursors_are_endpoint_and_filter_bound() {
+ let p = |v: &str| HashMap::from([("cursor".to_string(), v.to_string())]);
+ assert!(parse_cursor(&HashMap::new(), "search", "filters")
+ .unwrap()
+ .is_none());
+ let token = encode_cursor("search", "filters", json!({ "t": "x", "id": "y" }));
+ assert_eq!(
+ parse_cursor(&p(&token), "search", "filters").unwrap(),
+ Some(json!({ "t": "x", "id": "y" }))
+ );
+ assert!(parse_cursor(&p(&token), "records", "filters").is_err());
+ assert!(parse_cursor(&p(&token), "search", "different").is_err());
+ assert!(parse_cursor(&p("not-a-cursor"), "search", "filters").is_err());
+ }
+
/// The window parser refuses at the ROUTE, with the machine code the hosted
/// side emits — not just in the unit above.
#[tokio::test]
@@ -1703,6 +2161,13 @@ mod tests {
dir: &tempfile::TempDir,
policy: Option<crate::auth::PolicyStore>,
) -> Router {
+ seeded_router_and_db_with_policy(dir, policy).await.0
+ }
+
+ async fn seeded_router_and_db_with_policy(
+ dir: &tempfile::TempDir,
+ policy: Option<crate::auth::PolicyStore>,
+ ) -> (Router, Arc<DatabaseManager>) {
let db = Arc::new(
DatabaseManager::new(
dir.path().join("gateway.db").to_str().unwrap(),
@@ -1771,6 +2236,14 @@ mod tests {
.await
.unwrap();
}
+ for day in ["2026-07-21", "2026-07-22"] {
+ src.put_for_tests(
+ &format!("rollups/lic-1/org/{day}.json"),
+ serde_json::to_vec(&json!({ "day": day, "records": 1 })).unwrap(),
+ )
+ .await
+ .unwrap();
+ }
let ingestor = Ingestor::new(
src.clone() as Arc<dyn BlobSource>,
db.clone(),
@@ -1780,7 +2253,8 @@ mod tests {
.await
.unwrap();
ingestor.run_once().await.unwrap();
- router(db, src, "lic-1".to_string(), policy)
+ let app = router(db.clone(), src, "lic-1".to_string(), policy);
+ (app, db)
}
async fn get_json(router: &Router, uri: &str) -> (StatusCode, Value) {
@@ -1806,6 +2280,7 @@ mod tests {
)
.await;
assert_eq!(status, StatusCode::OK);
+ assert!(body.get("next_cursor").is_none());
let results = body["results"].as_array().unwrap();
// 2 frames + 2 audio + 2 memories match "roadmap"-adjacent tokens?
// "roadmap" appears in frame text, audio transcription, and memory
@@ -1848,6 +2323,22 @@ mod tests {
let mut sorted = ts.clone();
sorted.sort_by(|a, b| b.cmp(a));
assert_eq!(ts, sorted);
+
+ let (_, first) = get_json(
+ &router,
+ "/api/enterprise/v1/search?q=roadmap&since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1",
+ )
+ .await;
+ assert_eq!(first["result_count"], 1);
+ let cursor = first["next_cursor"].as_str().unwrap();
+ assert!(!cursor.bytes().all(|b| b.is_ascii_digit()));
+ let (_, second) = get_json(
+ &router,
+ &format!("/api/enterprise/v1/search?q=roadmap&since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1&cursor={cursor}"),
+ )
+ .await;
+ assert_eq!(second["result_count"], 1);
+ assert_ne!(first["results"][0], second["results"][0]);
}
#[tokio::test]
@@ -1883,6 +2374,7 @@ mod tests {
)
.await;
assert_eq!(status, StatusCode::OK);
+ assert!(body.get("next_cursor").is_none());
let records = body["records"].as_array().unwrap();
assert_eq!(records.len(), 2);
assert!(records.iter().all(|r| r["kind"] == "memory"));
@@ -1894,6 +2386,21 @@ mod tests {
sorted.sort();
assert_eq!(ts, sorted, "records are ascending by t");
assert_eq!(records[0]["memory_id"], 1);
+
+ let (_, first) = get_json(
+ &router,
+ "/api/enterprise/v1/records?since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1",
+ )
+ .await;
+ assert_eq!(first["record_count"], 1);
+ let cursor = first["next_cursor"].as_str().unwrap();
+ let (_, second) = get_json(
+ &router,
+ &format!("/api/enterprise/v1/records?since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1&cursor={cursor}"),
+ )
+ .await;
+ assert_eq!(second["record_count"], 1);
+ assert_ne!(first["records"][0], second["records"][0]);
}
#[tokio::test]
@@ -1935,11 +2442,126 @@ mod tests {
assert_eq!(status, StatusCode::OK);
assert_eq!(body["count"], 2);
let devices = body["devices"].as_array().unwrap();
- // Ordered by last_seen DESC — dev-b's records are newer.
assert_eq!(devices[0]["device_id"], "dev-b");
assert_eq!(devices[0]["label"], "bob-mbp");
+ assert!(devices[0]["member_email"].is_null());
assert!(devices[0]["platform"].is_null());
assert!(devices[0].get("last_seen").is_some());
+ assert!(body.get("next_cursor").is_none());
+
+ let (_, first) = get_json(
+ &router,
+ "/api/enterprise/v1/devices?pagination=keyset&limit=1",
+ )
+ .await;
+ assert_eq!(first["count"], 1);
+ assert_eq!(first["devices"][0]["device_id"], "dev-a");
+ let cursor = first["next_cursor"].as_str().unwrap();
+ let (_, second) = get_json(
+ &router,
+ &format!("/api/enterprise/v1/devices?pagination=keyset&limit=1&cursor={cursor}"),
+ )
+ .await;
+ assert_eq!(second["count"], 1);
+ assert_ne!(first["devices"][0], second["devices"][0]);
+ }
+
+ #[tokio::test]
+ async fn keyset_pages_do_not_shift_when_rows_change_between_requests() {
+ let dir = tempfile::tempdir().unwrap();
+ let router = seeded_router(&dir).await;
+ let (_, first) = get_json(
+ &router,
+ "/api/enterprise/v1/devices?pagination=keyset&limit=1",
+ )
+ .await;
+ assert_eq!(first["devices"][0]["device_id"], "dev-a");
+ let cursor = first["next_cursor"].as_str().unwrap();
+ let writable = sqlx::sqlite::SqlitePoolOptions::new()
+ .max_connections(1)
+ .connect_with(
+ sqlx::sqlite::SqliteConnectOptions::new()
+ .filename(dir.path().join("gateway.db"))
+ .read_only(false),
+ )
+ .await
+ .unwrap();
+
+ sqlx::query(
+ "INSERT INTO gateway_devices (device_id, device_label, enrolled_at, last_seen) \
+ VALUES ('dev-0', 'new-before-cursor', '2026-07-22T08:00:00Z', '2026-07-22T12:00:00Z')",
+ )
+ .execute(&writable)
+ .await
+ .unwrap();
+
+ let (_, second) = get_json(
+ &router,
+ &format!("/api/enterprise/v1/devices?pagination=keyset&limit=1&cursor={cursor}"),
+ )
+ .await;
+ assert_eq!(
+ second["devices"][0]["device_id"], "dev-b",
+ "an inserted row before the keyset must not repeat or shift dev-a"
+ );
+
+ let (_, first) = get_json(
+ &router,
+ "/api/enterprise/v1/records?since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1",
+ )
+ .await;
+ let first_record = first["records"][0].clone();
+ let cursor = first["next_cursor"].as_str().unwrap();
+ sqlx::query(
+ "UPDATE memories SET created_at = '2026-07-22T09:00:00Z' \
+ WHERE sync_modified_by = 'dev-b'",
+ )
+ .execute(&writable)
+ .await
+ .unwrap();
+ let (_, second) = get_json(
+ &router,
+ &format!("/api/enterprise/v1/records?since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1&cursor={cursor}"),
+ )
+ .await;
+ assert_ne!(
+ second["records"][0], first_record,
+ "moving a later row before the keyset must not duplicate page one"
+ );
+ }
+
+ #[tokio::test]
+ async fn rollups_return_continuation_cursors() {
+ let dir = tempfile::tempdir().unwrap();
+ let router = seeded_router(&dir).await;
+
+ let (_, legacy) = get_json(
+ &router,
+ "/api/enterprise/v1/rollups?from=2026-07-01&to=2026-07-31",
+ )
+ .await;
+ assert!(legacy.get("next_cursor").is_none());
+
+ let (status, first) = get_json(
+ &router,
+ "/api/enterprise/v1/rollups?from=2026-07-01&to=2026-07-31&pagination=keyset&limit=1",
+ )
+ .await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(first["count"], 1);
+ assert_eq!(first["rollups"][0]["day"], "2026-07-22");
+ let cursor = first["next_cursor"].as_str().unwrap();
+
+ let (_, second) = get_json(
+ &router,
+ &format!(
+ "/api/enterprise/v1/rollups?from=2026-07-01&to=2026-07-31&pagination=keyset&limit=1&cursor={cursor}"
+ ),
+ )
+ .await;
+ assert_eq!(second["count"], 1);
+ assert_eq!(second["rollups"][0]["day"], "2026-07-21");
+ assert!(second["next_cursor"].is_null());
}
#[tokio::test]
@@ -1957,6 +2579,32 @@ mod tests {
let key = body["files"][0]["key"].as_str().unwrap().to_string();
assert!(key.starts_with("enterprise-telemetry/lic-1/"));
+ let (_, first) = get_json(
+ &router,
+ "/api/enterprise/v1/files?since=2020-01-01T00:00:00Z&page_size=1",
+ )
+ .await;
+ let cursor = first["next_cursor"].as_str().unwrap();
+ assert!(!cursor.bytes().all(|b| b.is_ascii_digit()));
+ let (_, second) = get_json(
+ &router,
+ &format!(
+ "/api/enterprise/v1/files?since=2020-01-01T00:00:00Z&page_size=1&cursor={cursor}"
+ ),
+ )
+ .await;
+ assert_eq!(second["count"], 1);
+ assert_ne!(first["files"][0], second["files"][0]);
+
+ let (_, legacy_second) = get_json(
+ &router,
+ "/api/enterprise/v1/files?since=2020-01-01T00:00:00Z&page_size=1&cursor=1",
+ )
+ .await;
+ assert_eq!(legacy_second["count"], 1);
+ assert_eq!(legacy_second["files"][0], second["files"][0]);
+ assert!(legacy_second["next_cursor"].is_null());
+
let resp = router
.clone()
.oneshot(