YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
feat(gateway): add keyset pagination and device member email (#6173)
Ezra Ellette0dfd675f92dc
diff --git a/crates/screenpipe-gateway/e2e/conformance/VENDORED_FROM b/crates/screenpipe-gateway/e2e/conformance/VENDORED_FROMindex aed33711e..4f350f642 100644--- a/crates/screenpipe-gateway/e2e/conformance/VENDORED_FROM+++ b/crates/screenpipe-gateway/e2e/conformance/VENDORED_FROM@@ -1,7 +1,7 @@repo: screenpipe/website (PRIVATE)-branch: main-commit: c3fc73b43c496d86fad54ecff65a4e350d179a7c-date: 2026-08-11T16:43:39Z+branch: codex/enterprise-v1-pagination+commit: d2dd6427e9daff6b20aabedad1a211ee173a37a0+date: 2026-08-12T19:05:55Zfiles: openapi/enterprise-v1.yaml -> enterprise-v1.yamlconformance/spec.ts -> spec.tsconformance/cases.ts -> cases.tsdiff --git a/crates/screenpipe-gateway/e2e/conformance/cases.ts b/crates/screenpipe-gateway/e2e/conformance/cases.tsindex 24a0b5895..4b73abd5b 100644--- a/crates/screenpipe-gateway/e2e/conformance/cases.ts+++ b/crates/screenpipe-gateway/e2e/conformance/cases.ts@@ -54,33 +54,42 @@ const FILES_W = { since: SEEDED_WINDOW.since };export const cases: Case[] = [// ── /devices ─────────────────────────────────────────────────────────────{- id: "devices: both seeded devices, newest first",+ id: "devices: legacy shape and last-seen order remain unchanged",path: "/api/enterprise/v1/devices",expectStatus: 200,expect: (b) => {if (b.count !== EXPECTED.devices) throw new Error(`count=${b.count}`);- if (b.devices.length !== b.count) throw new Error("count disagrees with devices.length");+ if (b.devices.length !== b.count)+ throw new Error("count disagrees with devices.length");+ if ("next_cursor" in b)+ throw new Error("legacy response grew next_cursor");const ids = b.devices.map((d: any) => d.device_id).sort();- if (JSON.stringify(ids) !== JSON.stringify([...EXPECTED.deviceIds].sort())) {+ if (+ JSON.stringify(ids) !== JSON.stringify([...EXPECTED.deviceIds].sort())+ ) {throw new Error(`device_ids=${JSON.stringify(ids)}`);}- // Ordering is part of the contract: last_seen DESC on both, so bob (hour- // 10) precedes alice (hour 09) even though the two implementations derive- // last_seen from different sources (DEV-LIVENESS).if (b.devices[0].device_id !== "dev-bob") {- throw new Error(`expected dev-bob first, got ${b.devices[0].device_id}`);+ throw new Error(+ `expected dev-bob first, got ${b.devices[0].device_id}`,+ );}// Labels come from the wire format on the gateway and from the// enterprise_devices row hosted; the fixtures make them agree.- const byId = Object.fromEntries(b.devices.map((d: any) => [d.device_id, d]));- if (byId["dev-alice"].label !== "alice-mbp") throw new Error("alice label");- if (byId["dev-bob"].label !== "bob-thinkpad") throw new Error("bob label");+ const byId = Object.fromEntries(+ b.devices.map((d: any) => [d.device_id, d]),+ );+ if (byId["dev-alice"].label !== "alice-mbp")+ throw new Error("alice label");+ if (byId["dev-bob"].label !== "bob-thinkpad")+ throw new Error("bob label");},expectHosted: {why: "DEV-PLATFORM",assert: (b) => {const alice = b.devices.find((d: any) => d.device_id === "dev-alice");- if (alice.platform !== "macos") throw new Error(`platform=${alice.platform}`);+ if (alice.platform !== "macos")+ throw new Error(`platform=${alice.platform}`);},},expectGateway: {@@ -88,12 +97,51 @@ export const cases: Case[] = [assert: (b) => {for (const d of b.devices) {if (d.platform !== null || d.app_version !== null) {- throw new Error(`gateway must report null platform/app_version, got ${JSON.stringify(d)}`);+ throw new Error(+ `gateway must report null platform/app_version, got ${JSON.stringify(d)}`,+ );}}},},},+ {+ id: "devices: member email is hosted identity and nullable on the gateway",+ path: "/api/enterprise/v1/devices",+ expectStatus: 200,+ expectHosted: {+ why: "DEV-MEMBER-EMAIL",+ assert: (b) => {+ const alice = b.devices.find((d: any) => d.device_id === "dev-alice");+ if (alice.member_email !== "[email protected]") {+ throw new Error(`member_email=${alice.member_email}`);+ }+ },+ },+ expectGateway: {+ why: "DEV-MEMBER-EMAIL",+ assert: (b) => {+ for (const d of b.devices) {+ if (d.member_email !== null) {+ throw new Error(+ `gateway must report null member_email, got ${JSON.stringify(d)}`,+ );+ }+ }+ },+ },+ },+ {+ id: "devices: limit=1 returns a continuation cursor",+ path: "/api/enterprise/v1/devices",+ query: { pagination: "keyset", limit: "1" },+ expectStatus: 200,+ expect: (b) => {+ if (b.count !== 1) throw new Error(`limit ignored: ${b.count}`);+ if (b.next_cursor === null)+ throw new Error("second device was silently truncated");+ },+ },// ── /search ──────────────────────────────────────────────────────────────{@@ -103,10 +151,16 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {if (b.query !== "roadmap") throw new Error(`query echoed as ${b.query}`);- if (b.result_count !== b.results.length) throw new Error("result_count disagrees");- if (b.result_count === 0) throw new Error("no results — the window or the corpus is wrong");- if (b.window_hours !== 24) throw new Error(`window_hours=${b.window_hours}`);- if (b.device_id !== null || b.app_name !== null) throw new Error("unfiltered echo");+ if (b.result_count !== b.results.length)+ throw new Error("result_count disagrees");+ if ("next_cursor" in b)+ throw new Error("legacy response grew next_cursor");+ if (b.result_count === 0)+ throw new Error("no results — the window or the corpus is wrong");+ if (b.window_hours !== 24)+ throw new Error(`window_hours=${b.window_hours}`);+ if (b.device_id !== null || b.app_name !== null)+ throw new Error("unfiltered echo");const devices = new Set<string>(b.results.map((r: any) => r.device_id));for (const id of EXPECTED.deviceIds) {if (!devices.has(id)) {@@ -116,7 +170,9 @@ export const cases: Case[] = [const kinds = new Set<string>(b.results.map((r: any) => r.kind));for (const k of ["frame", "parsed", "audio", "memory"]) {if (!kinds.has(k)) {- throw new Error(`missing kind ${k} in ${Array.from(kinds).join(", ")}`);+ throw new Error(+ `missing kind ${k} in ${Array.from(kinds).join(", ")}`,+ );}}// Every result must be INSIDE the requested window on both targets. The@@ -127,33 +183,50 @@ export const cases: Case[] = [throw new Error(`result outside the window: ${r.t}`);}}+ const ts = b.results.map((r: any) => r.t);+ if (JSON.stringify(ts) !== JSON.stringify([...ts].sort().reverse())) {+ throw new Error(+ `search results must be newest-first: ${JSON.stringify(ts)}`,+ );+ }},expectHosted: {why: "SEARCH-ALGORITHM",assert: (b) => {- if (b.bytes_scanned <= 0) throw new Error("hosted must report a real byte count");- if (b.objects_scanned <= 0) throw new Error("hosted must report scanned objects");+ if (b.bytes_scanned <= 0)+ throw new Error("hosted must report a real byte count");+ if (b.objects_scanned <= 0)+ throw new Error("hosted must report scanned objects");},},expectGateway: {why: "SEARCH-ALGORITHM",assert: (b) => {- if (b.bytes_scanned !== 0 || b.objects_scanned !== 0 || b.truncated !== false) {+ if (+ b.bytes_scanned !== 0 ||+ b.objects_scanned !== 0 ||+ b.truncated !== false+ ) {throw new Error("the gateway has no byte-budget scan to describe, so these must be " +- `honestly 0/0/false — got ${b.bytes_scanned}/${b.objects_scanned}/${b.truncated}`+ `honestly 0/0/false — got ${b.bytes_scanned}/${b.objects_scanned}/${b.truncated}`,);}- // The gateway sorts results newest-first and the hosted API does not- // (SEARCH-ORDER). Pinned here so the gateway's guarantee, which is the- // stronger of the two, cannot silently regress.- const ts = b.results.map((r: any) => r.t);- if (JSON.stringify(ts) !== JSON.stringify([...ts].sort().reverse())) {- throw new Error(`the gateway must sort newest-first: ${JSON.stringify(ts)}`);- }},},},+ {+ id: "search: limit=1 returns a continuation cursor",+ path: "/api/enterprise/v1/search",+ query: { q: "roadmap", ...W, pagination: "keyset", limit: "1" },+ expectStatus: 200,+ expect: (b) => {+ if (b.result_count !== 1)+ throw new Error(`limit ignored: ${b.result_count}`);+ if (b.next_cursor === null)+ throw new Error("matching records were silently truncated");+ },+ },{id: "search: device_id scopes every kind",path: "/api/enterprise/v1/search",@@ -161,9 +234,11 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {if (b.device_id !== "dev-alice") throw new Error(`echo=${b.device_id}`);- if (b.result_count === 0) throw new Error("device filter returned nothing");+ if (b.result_count === 0)+ throw new Error("device filter returned nothing");for (const r of b.results) {- if (r.device_id !== "dev-alice") throw new Error(`leaked ${r.device_id}`);+ if (r.device_id !== "dev-alice")+ throw new Error(`leaked ${r.device_id}`);}},},@@ -173,7 +248,8 @@ export const cases: Case[] = [query: { q: "roadmap", since: "2020-01-01", until: "2020-01-02" },expectStatus: 200,expect: (b) => {- if (b.result_count !== 0 || b.results.length !== 0) throw new Error("expected empty");+ if (b.result_count !== 0 || b.results.length !== 0)+ throw new Error("expected empty");},},{@@ -184,10 +260,11 @@ export const cases: Case[] = [expect: (b) => {if (b.window_hours !== 24) {throw new Error(- `a bare date must be UTC midnight on both, giving a 24h window; got ${b.window_hours}`+ `a bare date must be UTC midnight on both, giving a 24h window; got ${b.window_hours}`,);}- if (b.result_count === 0) throw new Error("the July-22 window must contain the corpus");+ if (b.result_count === 0)+ throw new Error("the July-22 window must contain the corpus");},},{@@ -217,13 +294,20 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {if (b.kind !== "all") throw new Error(`kind echoed as ${b.kind}`);- if (b.record_count !== b.records.length) throw new Error("record_count disagrees");+ if (b.record_count !== b.records.length)+ throw new Error("record_count disagrees");if (b.record_count !== EXPECTED.recordsAllKinds) {- throw new Error(`expected ${EXPECTED.recordsAllKinds} records, got ${b.record_count}`);+ throw new Error(+ `expected ${EXPECTED.recordsAllKinds} records, got ${b.record_count}`,+ );}+ if ("next_cursor" in b)+ throw new Error("legacy response grew next_cursor");const ts = b.records.map((r: any) => r.t);if (JSON.stringify(ts) !== JSON.stringify([...ts].sort())) {- throw new Error(`records must be ascending by t: ${JSON.stringify(ts)}`);+ throw new Error(+ `records must be ascending by t: ${JSON.stringify(ts)}`,+ );}},},@@ -233,7 +317,8 @@ export const cases: Case[] = [query: { kind: "memory", ...W },expectStatus: 200,expect: (b) => {- if (b.record_count !== EXPECTED.memories) throw new Error(`count=${b.record_count}`);+ if (b.record_count !== EXPECTED.memories)+ throw new Error(`count=${b.record_count}`);for (const r of b.records) {if (r.kind !== "memory") throw new Error(`leaked kind ${r.kind}`);if (!r.content?.includes("gateway ships this quarter")) {@@ -241,7 +326,9 @@ export const cases: Case[] = [}if (r.memory_id !== 1) throw new Error(`memory_id=${r.memory_id}`);if (r.importance !== 0.8) throw new Error(`importance=${r.importance}`);- if (JSON.stringify(r.tags) !== JSON.stringify(["decision", "roadmap"])) {+ if (+ JSON.stringify(r.tags) !== JSON.stringify(["decision", "roadmap"])+ ) {throw new Error(`tags=${JSON.stringify(r.tags)}`);}}@@ -254,7 +341,8 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {if (b.kind !== "parsed") throw new Error(`kind=${b.kind}`);- if (b.record_count !== EXPECTED.parsed) throw new Error(`count=${b.record_count}`);+ if (b.record_count !== EXPECTED.parsed)+ throw new Error(`count=${b.record_count}`);for (const r of b.records) {if (r.kind !== "parsed") throw new Error(`leaked kind ${r.kind}`);if (r.app !== "Slack" || !r.text?.includes("structured update")) {@@ -269,8 +357,10 @@ export const cases: Case[] = [query: { kind: "nonsense", ...W },expectStatus: 200,expect: (b) => {- if (b.kind !== "nonsense") throw new Error("the kind must still be echoed");- if (b.record_count !== 0) throw new Error(`expected empty, got ${b.record_count}`);+ if (b.kind !== "nonsense")+ throw new Error("the kind must still be echoed");+ if (b.record_count !== 0)+ throw new Error(`expected empty, got ${b.record_count}`);},},{@@ -282,16 +372,20 @@ export const cases: Case[] = [// It is a case anyway because it pins the SHAPE of the empty answer, which// is what a consumer of the documented divergence relies on.expect: (b) => {- if (b.record_count !== 0) throw new Error(`expected empty, got ${b.record_count}`);+ if (b.record_count !== 0)+ throw new Error(`expected empty, got ${b.record_count}`);},},{id: "records: limit is respected",path: "/api/enterprise/v1/records",- query: { ...W, limit: "3" },+ query: { ...W, pagination: "keyset", limit: "3" },expectStatus: 200,expect: (b) => {- if (b.record_count !== 3) throw new Error(`limit ignored: ${b.record_count}`);+ if (b.record_count !== 3)+ throw new Error(`limit ignored: ${b.record_count}`);+ if (b.next_cursor === null)+ throw new Error("remaining records were silently truncated");},},@@ -302,8 +396,10 @@ export const cases: Case[] = [query: FILES_W,expectStatus: 200,expect: (b) => {- if (b.count !== b.files.length) throw new Error("count disagrees with files.length");- if (b.count !== EXPECTED.batchObjects) throw new Error(`count=${b.count}`);+ if (b.count !== b.files.length)+ throw new Error("count disagrees with files.length");+ if (b.count !== EXPECTED.batchObjects)+ throw new Error(`count=${b.count}`);for (const f of b.files) {if (!f.key.startsWith(`enterprise-telemetry/${LICENSE_ID}/`)) {throw new Error(`key escapes the org prefix: ${f.key}`);@@ -321,7 +417,8 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {if (b.count !== 1) throw new Error(`page_size ignored: ${b.count}`);- if (b.next_cursor === null) throw new Error("more objects exist, so next_cursor must be set");+ if (b.next_cursor === null)+ throw new Error("more objects exist, so next_cursor must be set");},},{@@ -331,7 +428,8 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {// This is THE case: 1 item hosted, 100 on the gateway, before the fix.- if (b.count !== 1) throw new Error(`expected the clamp to 1, got ${b.count}`);+ if (b.count !== 1)+ throw new Error(`expected the clamp to 1, got ${b.count}`);},},{@@ -350,7 +448,8 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {if (b.device_id !== "dev-alice") throw new Error(`echo=${b.device_id}`);- if (b.count !== 1) throw new Error(`expected 1 alice object, got ${b.count}`);+ if (b.count !== 1)+ throw new Error(`expected 1 alice object, got ${b.count}`);if (b.files[0].device_id !== "dev-alice") throw new Error("wrong device");},},@@ -397,10 +496,16 @@ export const cases: Case[] = [if (b.device !== "org") throw new Error(`device=${b.device}`);if (b.count !== b.rollups.length) throw new Error("count disagrees");if (b.count !== EXPECTED.rollups) throw new Error(`count=${b.count}`);+ if ("next_cursor" in b)+ throw new Error("legacy response grew next_cursor");const r = b.rollups[0];if (r.day !== "2026-07-22") throw new Error(`day=${r.day}`);- if (r.data.records !== 12) throw new Error(`data passed through wrong: ${JSON.stringify(r.data)}`);- if (JSON.stringify(r.data.devices) !== JSON.stringify(["dev-alice", "dev-bob"])) {+ if (r.data.records !== 12)+ throw new Error(`data passed through wrong: ${JSON.stringify(r.data)}`);+ if (+ JSON.stringify(r.data.devices) !==+ JSON.stringify(["dev-alice", "dev-bob"])+ ) {throw new Error(`data.devices=${JSON.stringify(r.data.devices)}`);}},@@ -412,7 +517,8 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {if (b.count !== 0) throw new Error(`expected empty, got ${b.count}`);- if (b.from !== "2020-01-01" || b.to !== "2020-01-02") throw new Error("bounds not echoed");+ if (b.from !== "2020-01-01" || b.to !== "2020-01-02")+ throw new Error("bounds not echoed");},},{@@ -426,7 +532,8 @@ export const cases: Case[] = [// because the intuitive expectation ("dev___etc") is wrong, and because a// surviving ".." only fails to be traversal since the value is a storage// key prefix rather than a filesystem path.- if (b.device !== "dev_.._etc") throw new Error(`sanitized to ${b.device}`);+ if (b.device !== "dev_.._etc")+ throw new Error(`sanitized to ${b.device}`);},},{@@ -448,7 +555,8 @@ export const cases: Case[] = [expectStatus: 200,expect: (b) => {if (b.kind !== "sop") throw new Error(`kind=${b.kind}`);- if (b.count !== b.entries.length) throw new Error("count disagrees with entries");+ if (b.count !== b.entries.length)+ throw new Error("count disagrees with entries");if (b.count !== 1) throw new Error(`count=${b.count}`);},},@@ -459,7 +567,8 @@ export const cases: Case[] = [targets: ["hosted"],expectStatus: 400,expect: (b) => {- if (!b.error?.includes("invalid kind")) throw new Error(`error=${b.error}`);+ if (!b.error?.includes("invalid kind"))+ throw new Error(`error=${b.error}`);// Every v1 400 carries a machine code. This one did not until the spec// declared BadRequest for it and the suite refused to validate the body —// which is the whole point of writing the contract down first.diff --git a/crates/screenpipe-gateway/e2e/conformance/enterprise-v1.yaml b/crates/screenpipe-gateway/e2e/conformance/enterprise-v1.yamlindex 2db4bf354..42ec4ebe5 100644--- a/crates/screenpipe-gateway/e2e/conformance/enterprise-v1.yaml+++ b/crates/screenpipe-gateway/e2e/conformance/enterprise-v1.yaml@@ -80,10 +80,29 @@ x-divergences:Neither field is in the wire format, so the gateway has no source forthem. Typed `[string, "null"]` on both rather than omitted, so a consumerwrites one branch instead of two.+ DEV-MEMBER-EMAIL:+ id: DEV-MEMBER-EMAIL+ operations: ["GET /devices"]+ fields: ["devices[].member_email"]+ hosted: >-+ enterprise_devices.member_email from the latest device heartbeat: the+ verified account email when member-authenticated, otherwise null.+ gateway: always null+ rationale: >-+ The customer-run gateway reads telemetry batches from object storage and+ has no account-identity channel. Keeping the field required and nullable+ gives consumers one stable response shape without inventing identity.SEARCH-ALGORITHM:id: SEARCH-ALGORITHMoperations: ["GET /search"]- fields: ["results", "records_scanned", "bytes_scanned", "objects_scanned", "truncated"]+ fields:+ [+ "results",+ "records_scanned",+ "bytes_scanned",+ "objects_scanned",+ "truncated",+ ]hosted: >-case-insensitive SUBSTRING match over a 256KB byte budget. `q=oadma`matches "roadmap". The three counters describe that scan and `truncated`@@ -97,26 +116,6 @@ x-divergences:Ingest-once is the gateway's reason to exist; throwing away the index toemulate a substring scan over blobs would make it slower AND worse. Thecounters are honestly zero rather than fabricated.- SEARCH-ORDER:- id: SEARCH-ORDER- operations: ["GET /search"]- fields: ["results"]- hosted: >-- results come back in SCAN order: objects newest-first, but lines in file- order within each object. So a single object's frame/parsed/audio/ui/memory rows- appear in the order the device wrote them, not by timestamp.- gateway: >-- results are sorted by `t` DESCENDING before truncation.- rationale: >-- NOT ruled on — found by this suite, not by the SCR-288 audit, and recorded- rather than silently converged because changing hosted result ordering is a- consumer-visible behavior change that needs an owner. The gateway's- guarantee is the stronger one and is pinned by the suite so it cannot- regress; the shared assertion is the weaker property both hold (every- result is inside the requested window). `/records` is NOT affected — both- sort ascending by `t` there.- TODO(owner): decide whether hosted /search should sort. If yes this entry- goes away and the assertion moves into the shared `expect`.SNAPSHOT-KIND:id: SNAPSHOT-KINDoperations: ["GET /records"]@@ -139,8 +138,8 @@ x-divergences:a page can come back count:0 with a non-null next_cursor. The cursor isthe storage backend's continuation token.gateway: >-- time-filters the whole listing then applies skip/take, so `count` is exact- and `next_cursor` is a decimal offset.+ time-filters the whole listing then applies a stable object-key keyset, so+ `count` is exact and inserts before the cursor do not shift later pages.rationale: >-Cursors are OPAQUE by contract and are not interchangeable across targets.A client must treat next_cursor as a token from the same target and must@@ -193,13 +192,21 @@ paths:operationId: listDevicessummary: List the devices enrolled under the bearer's organizationx-scope: read:devices- x-divergence: [DEV-LIVENESS, DEV-PLATFORM, TIMESTAMP-SERIALIZATION]+ x-divergence:+ [DEV-LIVENESS, DEV-PLATFORM, DEV-MEMBER-EMAIL, TIMESTAMP-SERIALIZATION]+ parameters:+ - $ref: "#/components/parameters/Pagination"+ - $ref: "#/components/parameters/Limit"+ - $ref: "#/components/parameters/Cursor"responses:"200":- description: The organization's devices, most recently seen first+ description: >-+ Devices in legacy last-seen order by default, or stable device-id+ order when pagination=keyset.content:application/json:schema: { $ref: "#/components/schemas/DeviceList" }+ "400": { $ref: "#/components/responses/BadRequest" }"401": { $ref: "#/components/responses/Unauthorized" }"403": { $ref: "#/components/responses/Forbidden" }"500": { $ref: "#/components/responses/ServerError" }@@ -209,7 +216,7 @@ paths:operationId: searchRecordssummary: Search telemetry recordsx-scope: read:search- x-divergence: [SEARCH-ALGORITHM, SEARCH-ORDER]+ x-divergence: [SEARCH-ALGORITHM]parameters:- $ref: "#/components/parameters/Q"- $ref: "#/components/parameters/DeviceId"@@ -217,7 +224,9 @@ paths:- $ref: "#/components/parameters/Since"- $ref: "#/components/parameters/Until"- $ref: "#/components/parameters/SinceHoursAgo"+ - $ref: "#/components/parameters/Pagination"- $ref: "#/components/parameters/Limit"+ - $ref: "#/components/parameters/Cursor"responses:"200":description: Matching records, newest first@@ -251,7 +260,9 @@ paths:- $ref: "#/components/parameters/Since"- $ref: "#/components/parameters/Until"- $ref: "#/components/parameters/SinceHoursAgo"+ - $ref: "#/components/parameters/Pagination"- $ref: "#/components/parameters/Limit"+ - $ref: "#/components/parameters/Cursor"responses:"200":description: Records in the window, oldest first@@ -389,6 +400,8 @@ paths:in: queryrequired: falseschema: { type: integer, default: 35, minimum: 1, maximum: 100 }+ - $ref: "#/components/parameters/Pagination"+ - $ref: "#/components/parameters/Cursor"responses:"200":description: Rollups in range, newest day first@@ -583,6 +596,24 @@ components:in: queryrequired: falseschema: { type: integer, default: 50, minimum: 1, maximum: 200 }+ Pagination:+ name: pagination+ in: query+ required: false+ schema: { type: string, enum: [keyset] }+ description: >-+ Set to `keyset` to opt into pagination and receive `next_cursor`.+ Omit it for the legacy response shape and ordering.+ Cursor:+ name: cursor+ in: query+ required: false+ schema: { type: string }+ description: >-+ OPAQUE continuation token from a previous response's `next_cursor`.+ Pass it back unchanged with the same filters; clients must not construct it.+ Search, records, devices, and rollups require `pagination=keyset` and+ use keyset state tied to a frozen scan boundary.responses:BadRequest:@@ -706,7 +737,12 @@ components:app: { type: [string, "null"] }window: { type: [string, "null"] }url: { type: [string, "null"] }- text: { type: [string, "null"], description: Frame OCR / accessibility text, or parsed-app projection text }+ text:+ {+ type: [string, "null"],+ description: Frame OCR / accessibility text,+ or parsed-app projection text,+ }transcription: { type: [string, "null"] }speaker: { type: [string, "null"] }content: { type: [string, "null"], description: 'kind:"memory" only' }@@ -721,12 +757,36 @@ components:Device:type: objectadditionalProperties: false- required: [device_id, label, platform, app_version, last_seen, enrolled_at]+ required:+ [+ device_id,+ label,+ member_email,+ platform,+ app_version,+ last_seen,+ enrolled_at,+ ]properties:device_id: { type: string }- label: { type: string, description: Hostname when known, else the device id }- platform: { type: [string, "null"], description: Always null on the gateway (DEV-PLATFORM) }- app_version: { type: [string, "null"], description: Always null on the gateway (DEV-PLATFORM) }+ label:+ { type: string, description: Hostname when known, else the device id }+ member_email:+ type: [string, "null"]+ format: email+ description: >-+ Verified account email carried by the latest member-authenticated+ heartbeat; always null on the gateway (DEV-MEMBER-EMAIL).+ platform:+ {+ type: [string, "null"],+ description: Always null on the gateway (DEV-PLATFORM),+ }+ app_version:+ {+ type: [string, "null"],+ description: Always null on the gateway (DEV-PLATFORM),+ }last_seen:type: [string, "null"]format: date-time@@ -745,6 +805,9 @@ components:devices:type: arrayitems: { $ref: "#/components/schemas/Device" }+ next_cursor:+ type: [string, "null"]+ description: Present only with pagination=keyset; null on the final page.SearchResult:type: object@@ -769,13 +832,34 @@ components:minimum: 0description: Effective response window; use the `since_hours_ago` query parameter to request a relative window.records_scanned: { type: integer, minimum: 0 }- bytes_scanned: { type: integer, minimum: 0, description: Always 0 on the gateway (SEARCH-ALGORITHM) }- objects_scanned: { type: integer, minimum: 0, description: Always 0 on the gateway (SEARCH-ALGORITHM) }- truncated: { type: boolean, description: Always false on the gateway (SEARCH-ALGORITHM) }+ bytes_scanned:+ {+ type: integer,+ minimum: 0,+ description: Always 0 on the gateway (SEARCH-ALGORITHM),+ }+ objects_scanned:+ {+ type: integer,+ minimum: 0,+ description: Always 0 on the gateway (SEARCH-ALGORITHM),+ }+ truncated:+ {+ type: boolean,+ description: Always false on the gateway (SEARCH-ALGORITHM),+ }result_count: { type: integer, minimum: 0 }results:type: arrayitems: { $ref: "#/components/schemas/RecordSummary" }+ next_cursor:+ type: [string, "null"]+ description: >-+ Present only with pagination=keyset. Mutation-safe token for more matches+ in the frozen scan window. A null+ cursor with truncated:true means the byte budget, not pagination, ended+ the scan; narrow the query or time window.RecordList:type: object@@ -791,7 +875,12 @@ components:- recordsproperties:device_id: { type: [string, "null"] }- kind: { type: string, description: Echoed back lower-cased, including an unrecognized one }+ kind:+ {+ type: string,+ description: Echoed back lower-cased,+ including an unrecognized one,+ }window_hours:type: numberminimum: 0@@ -803,6 +892,13 @@ components:records:type: arrayitems: { $ref: "#/components/schemas/RecordSummary" }+ next_cursor:+ type: [string, "null"]+ description: >-+ Present only with pagination=keyset. Mutation-safe token for more records+ in the frozen scan window. A null+ cursor with truncated:true means the byte budget, not pagination, ended+ the scan; narrow the time window.FileEntry:type: object@@ -810,12 +906,21 @@ components:required: [key, size, last_modified, device_id]properties:key: { type: string }- size: { type: [integer, "null"], minimum: 0, description: Nullable on the gateway (FILES-NULLABILITY) }+ size:+ {+ type: [integer, "null"],+ minimum: 0,+ description: Nullable on the gateway (FILES-NULLABILITY),+ }last_modified:type: [string, "null"]format: date-timedescription: Nullable on the gateway (FILES-NULLABILITY)- device_id: { type: string, description: '"unknown" when the key has no device segment' }+ device_id:+ {+ type: string,+ description: '"unknown" when the key has no device segment',+ }FileList:type: object@@ -864,6 +969,9 @@ components:rollups:type: arrayitems: { $ref: "#/components/schemas/Rollup" }+ next_cursor:+ type: [string, "null"]+ description: Present only with pagination=keyset; null on the final page.ManifestEntry:type: object@@ -883,7 +991,8 @@ components:- keyproperties:artifact_id: { type: string }- kind: { type: string, enum: [sop, skill, memory, chart, trajectory, intel] }+ kind:+ { type: string, enum: [sop, skill, memory, chart, trajectory, intel] }type: { type: string }title: { type: string }status: { type: string }@@ -925,7 +1034,12 @@ components:additionalProperties: falserequired: [name, prompt_body]properties:- name: { type: string, pattern: "^[a-zA-Z0-9][a-zA-Z0-9_-]*$", maxLength: 100 }+ name:+ {+ type: string,+ pattern: "^[a-zA-Z0-9][a-zA-Z0-9_-]*$",+ maxLength: 100,+ }prompt_body: { type: string, maxLength: 262144 }display_name: { type: string }schedule: { type: string, default: every 30m }diff --git a/crates/screenpipe-gateway/e2e/conformance/fixtures/devices.json b/crates/screenpipe-gateway/e2e/conformance/fixtures/devices.jsonindex df39db583..b432daa57 100644--- a/crates/screenpipe-gateway/e2e/conformance/fixtures/devices.json+++ b/crates/screenpipe-gateway/e2e/conformance/fixtures/devices.json@@ -9,8 +9,10 @@"gateway (divergence DEV-PLATFORM): neither field is in the wire format, so","there is nothing for a gateway to read them from. `last_seen_at` is","heartbeat liveness here and MAX(record timestamp) there (DEV-LIVENESS).",- "Both are recorded divergences, so the suite asserts their TYPE and not",- "their value.",+ "`member_email` is the hosted heartbeat identity and is ALWAYS null on the",+ "gateway (DEV-MEMBER-EMAIL), which has no account-identity channel.",+ "The contract keeps every field present and nullable; target-specific cases",+ "pin the values available on each side.","","`last_seen_at` sits inside the seeded July-22 window so a hosted /devices","response is ordered the same way a gateway one is (bob newer than alice)."@@ -19,6 +21,7 @@{"device_id": "dev-bob","hostname": "bob-thinkpad",+ "member_email": "[email protected]","platform": "windows","app_version": "2.5.136","last_seen_at": "2026-07-22T10:08:00+00:00",@@ -27,6 +30,7 @@{"device_id": "dev-alice","hostname": "alice-mbp",+ "member_email": "[email protected]","platform": "macos","app_version": "2.5.136","last_seen_at": "2026-07-22T09:08:00+00:00",diff --git a/crates/screenpipe-gateway/e2e/conformance/fixtures/index.ts b/crates/screenpipe-gateway/e2e/conformance/fixtures/index.tsindex 534c4b938..b5e698f05 100644--- a/crates/screenpipe-gateway/e2e/conformance/fixtures/index.ts+++ b/crates/screenpipe-gateway/e2e/conformance/fixtures/index.ts@@ -47,6 +47,7 @@ export const NON_SK_ENT_TOKEN: string = tokensFile.non_sk_ent_token;export const DEVICE_ROWS: Array<{device_id: string;hostname: string;+ member_email: string | null;platform: string;app_version: string;last_seen_at: string;diff --git a/crates/screenpipe-gateway/e2e/conformance/run.test.ts b/crates/screenpipe-gateway/e2e/conformance/run.test.tsindex f7b5a4e69..a971959ba 100644--- a/crates/screenpipe-gateway/e2e/conformance/run.test.ts+++ b/crates/screenpipe-gateway/e2e/conformance/run.test.ts@@ -21,22 +21,31 @@import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";-vi.mock("@/lib/supabase", async () => (await import("./targets/hosted-inproc")).supabaseMock());+vi.mock("@/lib/supabase", async () =>+ (await import("./targets/hosted-inproc")).supabaseMock(),+);vi.mock("@/lib/enterprise/storage", async () =>- (await import("./targets/hosted-inproc")).storageMock()+ (await import("./targets/hosted-inproc")).storageMock(),+);+vi.mock("@/lib/enterprise/audit", async () =>+ (await import("./targets/hosted-inproc")).auditMock(),);-vi.mock("@/lib/enterprise/audit", async () => (await import("./targets/hosted-inproc")).auditMock());vi.mock("@/lib/enterprise/artifacts/store", async () =>- (await import("./targets/hosted-inproc")).artifactsStoreMock()+ (await import("./targets/hosted-inproc")).artifactsStoreMock(),);vi.mock("@/lib/enterprise/auth", async () =>- (await import("./targets/hosted-inproc")).enterpriseAuthMock()+ (await import("./targets/hosted-inproc")).enterpriseAuthMock(),);vi.mock("@clerk/backend", () => ({ verifyToken: vi.fn() }));vi.mock("@clerk/nextjs/server", () => ({ clerkClient: vi.fn() }));import { cases, type Case } from "./cases";-import { ALL_SCOPES_TOKEN, EXPECTED, LICENSE_ID, SEEDED_WINDOW } from "./fixtures";+import {+ ALL_SCOPES_TOKEN,+ EXPECTED,+ LICENSE_ID,+ SEEDED_WINDOW,+} from "./fixtures";import {contentTypeFor,operationFor,@@ -58,7 +67,9 @@ beforeAll(() => {process.env.SUPABASE_SERVICE_KEY ||= "conformance-service-key";});-async function send(req: Parameters<typeof import("./targets/http").request>[0]) {+async function send(+ req: Parameters<typeof import("./targets/http").request>[0],+) {if (activeTarget() === "http") {return (await import("./targets/http")).request(req);}@@ -72,7 +83,8 @@ async function send(req: Parameters<typeof import("./targets/http").request>[0])* catch-all routes are templated in the spec and concrete in a case.*/function specPathFor(path: string): string {- if (path.startsWith("/api/enterprise/v1/files/")) return "/api/enterprise/v1/files/{key}";+ if (path.startsWith("/api/enterprise/v1/files/"))+ return "/api/enterprise/v1/files/{key}";if (path.startsWith("/api/enterprise/v1/frames/")) {return "/api/enterprise/v1/frames/{device_id}/{frame_id}";}@@ -101,20 +113,25 @@ async function check(c: Case) {const raw = await res.clone().text();expect(res.status,- `${method} ${c.path} -> ${res.status}, body: ${raw.slice(0, 400)}`+ `${method} ${c.path} -> ${res.status}, body: ${raw.slice(0, 400)}`,).toBe(c.expectStatus);// 2. Content type, exactly as the spec declares it for this status.const declared = contentTypeFor(method, specPath, c.expectStatus);const actual = (res.headers.get("content-type") ?? "").split(";")[0].trim();- expect(actual, `${method} ${c.path} ${c.expectStatus} content-type`).toBe(declared);+ expect(actual, `${method} ${c.path} ${c.expectStatus} content-type`).toBe(+ declared,+ );// 3. Required response headers.for (const [name, schema] of Object.entries<any>(- requiredHeaders(method, specPath, c.expectStatus)+ requiredHeaders(method, specPath, c.expectStatus),)) {const value = res.headers.get(name);- expect(value, `${method} ${c.path}: required header '${name}' is missing`).not.toBeNull();+ expect(+ value,+ `${method} ${c.path}: required header '${name}' is missing`,+ ).not.toBeNull();if (schema.schema?.const !== undefined) {expect(value, `header '${name}'`).toBe(schema.schema.const);}@@ -129,7 +146,7 @@ async function check(c: Case) {expect(ok,`${method} ${c.path} ${c.expectStatus} does not match the spec:\n${validate.errorText()}\n` +- `body: ${JSON.stringify(body).slice(0, 600)}`+ `body: ${JSON.stringify(body).slice(0, 600)}`,).toBe(true);}@@ -140,7 +157,9 @@ async function check(c: Case) {return { res, body, raw };}-const applicable = cases.filter((c) => (c.targets ?? ["hosted", "gateway"]).includes(impl));+const applicable = cases.filter((c) =>+ (c.targets ?? ["hosted", "gateway"]).includes(impl),+);describe(`conformance [${activeTarget()} / impl=${impl}]`, () => {it("has cases to run (an empty run must never look like a pass)", () => {@@ -154,6 +173,106 @@ describe(`conformance [${activeTarget()} / impl=${impl}]`, () => {}});+describe(`keyset pagination [${impl}]`, () => {+ it.each([+ [+ "devices",+ "/api/enterprise/v1/devices",+ { pagination: "keyset", limit: "1" },+ "devices",+ ],+ [+ "search",+ "/api/enterprise/v1/search",+ {+ q: "roadmap",+ since: SEEDED_WINDOW.since,+ until: SEEDED_WINDOW.until,+ pagination: "keyset",+ limit: "1",+ },+ "results",+ ],+ [+ "records",+ "/api/enterprise/v1/records",+ {+ since: SEEDED_WINDOW.since,+ until: SEEDED_WINDOW.until,+ pagination: "keyset",+ limit: "3",+ },+ "records",+ ],+ [+ "files",+ "/api/enterprise/v1/files",+ { since: SEEDED_WINDOW.since, page_size: "1" },+ "files",+ ],+ ])(+ "%s follows the returned opaque cursor without repeating a row",+ async (name, path, query, field) => {+ const first = await send({ path, query, headers: authHeader() });+ expect(first.status, await first.clone().text()).toBe(200);+ const firstBody = await first.json();+ expect(firstBody.next_cursor).toEqual(expect.any(String));+ if (name !== "files" || impl === "gateway") {+ expect(firstBody.next_cursor).not.toMatch(/^\d+$/);+ }++ const second = await send({+ path,+ query: { ...query, cursor: firstBody.next_cursor },+ headers: authHeader(),+ });+ expect(second.status, await second.clone().text()).toBe(200);+ const secondBody = await second.json();+ const identity = (row: any) =>+ row.kind === undefined+ ? (row.key ?? row.device_id)+ : [+ row.device_id,+ row.kind,+ row.t,+ row.frame_id,+ row.memory_id,+ row.text,+ row.transcription,+ ].join("|");+ expect(firstBody[field].map(identity)).not.toContain(+ secondBody[field][0] && identity(secondBody[field][0]),+ );+ },+ );++ it("binds a cursor to its original filters", async () => {+ const first = await send({+ path: "/api/enterprise/v1/search",+ query: {+ q: "roadmap",+ ...SEEDED_WINDOW,+ pagination: "keyset",+ limit: "1",+ },+ headers: authHeader(),+ });+ const body = await first.json();+ const reused = await send({+ path: "/api/enterprise/v1/search",+ query: {+ q: "different",+ ...SEEDED_WINDOW,+ pagination: "keyset",+ limit: "1",+ cursor: body.next_cursor,+ },+ headers: authHeader(),+ });+ expect(reused.status).toBe(400);+ });+});+/*** The two binary routes need a key/id discovered from a previous response, so* they cannot be static table rows. Their 404 and validation cases ARE in the@@ -171,9 +290,14 @@ describe(`binary routes [${impl}]`, () => {expect(files.length).toBe(EXPECTED.batchObjects);const key: string = files[0].key;- const res = await send({ path: `/api/enterprise/v1/files/${key}`, headers: authHeader() });+ const res = await send({+ path: `/api/enterprise/v1/files/${key}`,+ headers: authHeader(),+ });expect(res.status, await res.clone().text()).toBe(200);- expect((res.headers.get("content-type") ?? "").split(";")[0]).toBe("application/x-ndjson");+ expect((res.headers.get("content-type") ?? "").split(";")[0]).toBe(+ "application/x-ndjson",+ );// The spec marks this header required — it is how a consumer knows WHICH// object it got when it followed a cursor.expect(res.headers.get("x-screenpipe-source")).toBe(key);@@ -183,7 +307,9 @@ describe(`binary routes [${impl}]`, () => {const lines = text.trim().split("\n");expect(lines.length).toBe(7); // the six in-window records + one stale recordconst kinds = lines.map((l) => JSON.parse(l).kind);- expect(new Set(kinds)).toEqual(new Set(["frame", "parsed", "audio", "ui", "memory"]));+ expect(new Set(kinds)).toEqual(+ new Set(["frame", "parsed", "audio", "ui", "memory"]),+ );for (const line of lines) {const rec = JSON.parse(line);expect(EXPECTED.deviceIds).toContain(rec.device_id);@@ -202,12 +328,14 @@ describe(`binary routes [${impl}]`, () => {headers: authHeader(),});expect(res.status, await res.clone().text()).toBe(200);- expect((res.headers.get("content-type") ?? "").split(";")[0]).toBe("image/jpeg");+ expect((res.headers.get("content-type") ?? "").split(";")[0]).toBe(+ "image/jpeg",+ );expect(res.headers.get("cache-control")).toBe("private, max-age=3600");const bytes = new Uint8Array(await res.arrayBuffer());// A real JPEG SOI marker, so a JSON error body cannot pass as an image.expect([bytes[0], bytes[1]]).toEqual([0xff, 0xd8]);- }+ },);});@@ -218,43 +346,62 @@ describe(`binary routes [${impl}]`, () => {* This is the SCR-288 divergence-11 fix, asserted at the surface rather than the* unit: every content scope refuses, and the two hand-rolled routes refuse too.*/-describe.runIf(activeTarget() === "hosted-inproc")("write-only archive (hosted only)", () => {- beforeEach(async () => {- const { posture } = await import("./targets/hosted-inproc");- posture.writeOnlyArchive = true;- });+describe.runIf(activeTarget() === "hosted-inproc")(+ "write-only archive (hosted only)",+ () => {+ beforeEach(async () => {+ const { posture } = await import("./targets/hosted-inproc");+ posture.writeOnlyArchive = true;+ });- const contentRoutes: Array<[string, string, Record<string, string> | undefined]> = [- ["GET", "/api/enterprise/v1/search", { q: "roadmap" }],- ["GET", "/api/enterprise/v1/records", undefined],- ["GET", "/api/enterprise/v1/files", undefined],- ["GET", "/api/enterprise/v1/rollups", undefined],- ["GET", "/api/enterprise/v1/workflows/generated", undefined],- [- "GET",- `/api/enterprise/v1/files/enterprise-telemetry/${LICENSE_ID}/dev-alice/direct/batch-alice.jsonl`,- undefined,- ],- ["GET", "/api/enterprise/v1/frames/dev-alice/1", undefined],- ];+ const contentRoutes: Array<+ [string, string, Record<string, string> | undefined]+ > = [+ ["GET", "/api/enterprise/v1/search", { q: "roadmap" }],+ ["GET", "/api/enterprise/v1/records", undefined],+ ["GET", "/api/enterprise/v1/files", undefined],+ ["GET", "/api/enterprise/v1/rollups", undefined],+ ["GET", "/api/enterprise/v1/workflows/generated", undefined],+ [+ "GET",+ `/api/enterprise/v1/files/enterprise-telemetry/${LICENSE_ID}/dev-alice/direct/batch-alice.jsonl`,+ undefined,+ ],+ ["GET", "/api/enterprise/v1/frames/dev-alice/1", undefined],+ ];- it.each(contentRoutes)("%s %s refuses with a typed 409", async (method, path, query) => {- const { request } = await import("./targets/hosted-inproc");- const res = await request({ method, path, query, headers: authHeader() });- const raw = await res.clone().text();- expect(res.status, `${path} -> ${res.status}: ${raw.slice(0, 300)}`).toBe(409);- const validate = validatorFor(method, specPathFor(path), 409)!;- const body = JSON.parse(raw);- expect(validate(body), validate.errorText()).toBe(true);- expect(body.code).toBe("write_only_archive");- expect(body.surface).toBe("hosted_api");- // The whole point of the denial is to route the caller somewhere useful.- expect(body.gateway_url).toBeTruthy();- });+ it.each(contentRoutes)(+ "%s %s refuses with a typed 409",+ async (method, path, query) => {+ const { request } = await import("./targets/hosted-inproc");+ const res = await request({+ method,+ path,+ query,+ headers: authHeader(),+ });+ const raw = await res.clone().text();+ expect(+ res.status,+ `${path} -> ${res.status}: ${raw.slice(0, 300)}`,+ ).toBe(409);+ const validate = validatorFor(method, specPathFor(path), 409)!;+ const body = JSON.parse(raw);+ expect(validate(body), validate.errorText()).toBe(true);+ expect(body.code).toBe("write_only_archive");+ expect(body.surface).toBe("hosted_api");+ // The whole point of the denial is to route the caller somewhere useful.+ expect(body.gateway_url).toBeTruthy();+ },+ );- it("does NOT refuse /devices — fleet metadata is control-plane, not content", async () => {- const { request } = await import("./targets/hosted-inproc");- const res = await request({ path: "/api/enterprise/v1/devices", headers: authHeader() });- expect(res.status).toBe(200);- });-});+ it("does NOT refuse /devices — fleet metadata is control-plane, not content", async () => {+ const { request } = await import("./targets/hosted-inproc");+ const res = await request({+ path: "/api/enterprise/v1/devices",+ headers: authHeader(),+ });+ expect(res.status).toBe(200);+ });+ },+);diff --git a/crates/screenpipe-gateway/e2e/conformance/strictness.test.ts b/crates/screenpipe-gateway/e2e/conformance/strictness.test.tsindex eb3ffba07..a97badb0b 100644--- a/crates/screenpipe-gateway/e2e/conformance/strictness.test.ts+++ b/crates/screenpipe-gateway/e2e/conformance/strictness.test.ts@@ -61,6 +61,7 @@ const goodDeviceList = () => ({{device_id: "dev-alice",label: "alice-mbp",+ member_email: "[email protected]",platform: "macos",app_version: "2.5.136",last_seen: "2026-07-22T09:08:00+00:00",@@ -69,6 +70,7 @@ const goodDeviceList = () => ({{device_id: "dev-bob",label: "bob-thinkpad",+ member_email: null,platform: null,app_version: null,last_seen: "2026-07-22T10:08:00Z",@@ -125,6 +127,16 @@ describe("the good bodies validate (or every rejection below proves nothing)", (const v = componentValidator(name as string);expect(v(body), v.errorText()).toBe(true);});++ it.each([+ ["DeviceList", goodDeviceList()],+ ["SearchResult", goodSearchResult()],+ ["RollupList", goodRollupList()],+ ])("%s also accepts the opt-in pagination field", (name, body: any) => {+ body.next_cursor = null;+ const v = componentValidator(name);+ expect(v(body), v.errorText()).toBe(true);+ });});describe("renaming a required field is REJECTED", () => {@@ -167,7 +179,7 @@ describe("renaming a required field is REJECTED", () => {body.files[0][`x_${field}`] = body.files[0][field];delete body.files[0][field];expect(componentValidator("FileList")(body)).toBe(false);- }+ },);});@@ -263,7 +275,9 @@ describe("nulling a non-nullable field is REJECTED", () => {describe("the nullable fields ARE allowed to be null (accepted divergences)", () => {it("every RecordSummary field except kind", () => {- const nullable = Object.keys(goodRecordSummary()).filter((k) => k !== "kind");+ const nullable = Object.keys(goodRecordSummary()).filter(+ (k) => k !== "kind",+ );for (const field of nullable) {const body: any = goodRecordSummary();body[field] = null;@@ -279,6 +293,12 @@ describe("the nullable fields ARE allowed to be null (accepted divergences)", ()expect(componentValidator("DeviceList")(body)).toBe(true);});+ it("Device.member_email — null when no verified identity source exists (DEV-MEMBER-EMAIL)", () => {+ const body: any = goodDeviceList();+ body.devices[0].member_email = null;+ expect(componentValidator("DeviceList")(body)).toBe(true);+ });+it("FileEntry.size / last_modified — nullable on the gateway (FILES-NULLABILITY)", () => {const body: any = goodFileList();body.files[0].size = null;@@ -288,10 +308,22 @@ describe("the nullable fields ARE allowed to be null (accepted divergences)", ()});describe("removing a required field is REJECTED", () => {+ it("Device.member_email", () => {+ const body: any = goodDeviceList();+ delete body.devices[0].member_email;+ expect(componentValidator("DeviceList")(body)).toBe(false);+ });+it.each([["DeviceList", ["count", "devices"]],- ["SearchResult", ["query", "result_count", "results", "truncated", "window_hours"]],- ["FileList", ["device_id", "count", "files", "next_cursor", "window_hours"]],+ [+ "SearchResult",+ ["query", "result_count", "results", "truncated", "window_hours"],+ ],+ [+ "FileList",+ ["device_id", "count", "files", "next_cursor", "window_hours"],+ ],["RollupList", ["ok", "device", "from", "to", "count", "rollups"]],])("%s", (name, fields) => {for (const field of fields as string[]) {@@ -304,7 +336,7 @@ describe("removing a required field is REJECTED", () => {delete body[field];expect(componentValidator(name as string)(body),- `${name} validated without required '${field}'`+ `${name} validated without required '${field}'`,).toBe(false);}});@@ -313,7 +345,9 @@ describe("removing a required field is REJECTED", () => {describe("error bodies are strict too", () => {it("{error} accepts only that", () => {expect(componentValidator("Error")({ error: "not found" })).toBe(true);- expect(componentValidator("Error")({ error: "x", stack: "..." })).toBe(false);+ expect(componentValidator("Error")({ error: "x", stack: "..." })).toBe(+ false,+ );expect(componentValidator("Error")({})).toBe(false);});@@ -333,17 +367,23 @@ describe("error bodies are strict too", () => {it("the write-only denial code and surface are pinned", () => {const v = componentValidator("WriteOnlyDenial");- expect(v({ error: "x", code: "write_only_archive", surface: "hosted_api" })).toBe(true);+ expect(+ v({ error: "x", code: "write_only_archive", surface: "hosted_api" }),+ ).toBe(true);expect(v({error: "x",code: "write_only_archive",surface: "hosted_api",gateway_url: "https://gw",- })+ }),).toBe(true);- expect(v({ error: "x", code: "write_only", surface: "hosted_api" })).toBe(false);- expect(v({ error: "x", code: "write_only_archive", surface: "made_up" })).toBe(false);+ expect(v({ error: "x", code: "write_only", surface: "hosted_api" })).toBe(+ false,+ );+ expect(+ v({ error: "x", code: "write_only_archive", surface: "made_up" }),+ ).toBe(false);});});@@ -351,17 +391,25 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {it("every JSON response object schema sets additionalProperties: false", () => {// The one deliberate exception is documented in the spec: a rollup's `data`// and an artifact body are passthroughs whose shape the route does not own.- const allowed = new Set(["Rollup.data", "GeneratedWorkflows.artifacts.items"]);+ const allowed = new Set([+ "Rollup.data",+ "GeneratedWorkflows.artifacts.items",+ ]);const offenders: string[] = [];walk(rawSpec().components.schemas, "");function walk(node: any, path: string) {if (!node || typeof node !== "object") return;- if (Array.isArray(node)) return node.forEach((n, i) => walk(n, `${path}[${i}]`));+ if (Array.isArray(node))+ return node.forEach((n, i) => walk(n, `${path}[${i}]`));const isObjectSchema =node.type === "object" ||(Array.isArray(node.type) && node.type.includes("object")) ||!!node.properties;- if (isObjectSchema && node.additionalProperties !== false && !allowed.has(path)) {+ if (+ isObjectSchema &&+ node.additionalProperties !== false &&+ !allowed.has(path)+ ) {offenders.push(path || "<root>");}for (const [k, v] of Object.entries(node)) {@@ -374,7 +422,10 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {}}}- expect(offenders, `these object schemas would validate a renamed field`).toEqual([]);+ expect(+ offenders,+ `these object schemas would validate a renamed field`,+ ).toEqual([]);});it("every RESPONSE object schema has an exhaustive required list", () => {@@ -402,10 +453,17 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {}expect(requestSchemas.size,- "no request schemas found — the derivation broke and this test is now vacuous"+ "no request schemas found — the derivation broke and this test is now vacuous",).toBeGreaterThan(0);- const optionalByDesign = new Set(["WriteOnlyDenial.gateway_url"]);+ const optionalByDesign = new Set([+ "WriteOnlyDenial.gateway_url",+ // Pagination is opt-in so legacy responses retain their exact shape.+ "DeviceList.next_cursor",+ "SearchResult.next_cursor",+ "RecordList.next_cursor",+ "RollupList.next_cursor",+ ]);const gaps: string[] = [];for (const [name, schema] of Object.entries<any>(doc.components.schemas)) {if (!schema.properties) continue;@@ -418,7 +476,10 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {}}}- expect(gaps, "these properties could vanish without the schema noticing").toEqual([]);+ expect(+ gaps,+ "these properties could vanish without the schema noticing",+ ).toEqual([]);});it("every operation declares an x-scope, or is an explicitly public route", () => {@@ -426,7 +487,7 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {expect(op.scope !== null || op.isPublic,`${op.method} ${op.path} has neither x-scope nor x-public: true — the ` +- `auth matrix cannot classify it, so it would be silently unasserted`+ `auth matrix cannot classify it, so it would be silently unasserted`,).toBe(true);}});@@ -437,7 +498,7 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {for (const id of op.divergences) {expect(Object.keys(ledger),- `${op.method} ${op.path} claims divergence '${id}'`+ `${op.method} ${op.path} claims divergence '${id}'`,).toContain(id);}}@@ -447,19 +508,22 @@ describe("the spec itself stays strict — a relaxation fails HERE", () => {for (const [id, entry] of Object.entries<any>(divergenceLedger())) {expect(entry.rationale, `${id} has no rationale`).toBeTruthy();expect(entry.hosted, `${id} does not say what hosted does`).toBeTruthy();- expect(entry.gateway, `${id} does not say what the gateway does`).toBeTruthy();+ expect(+ entry.gateway,+ `${id} does not say what the gateway does`,+ ).toBeTruthy();}});it("validatorFor THROWS for a status the spec does not describe", () => {- expect(() => validatorFor("GET", "/api/enterprise/v1/devices", 418)).toThrow(- /does not define a 418 response/- );+ expect(() =>+ validatorFor("GET", "/api/enterprise/v1/devices", 418),+ ).toThrow(/does not define a 418 response/);});it("validatorFor THROWS for an operation the spec does not describe", () => {- expect(() => validatorFor("GET", "/api/enterprise/v1/experimental", 200)).toThrow(- /does not describe/- );+ expect(() =>+ validatorFor("GET", "/api/enterprise/v1/experimental", 200),+ ).toThrow(/does not describe/);});});diff --git a/crates/screenpipe-gateway/src/api.rs b/crates/screenpipe-gateway/src/api.rsindex 32093cc4c..f83f8a3ed 100644--- a/crates/screenpipe-gateway/src/api.rs+++ b/crates/screenpipe-gateway/src/api.rs@@ -60,10 +60,12 @@ use axum::{routing::{any, get, MethodRouter},Json, Router,};+use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _};use chrono::{DateTime, Duration as ChronoDuration, SecondsFormat, Utc};use screenpipe_db::DatabaseManager;use screenpipe_sync::{BlobSource, ListRequest};use screenpipe_telemetry_wire::{frame_image_key, org_telemetry_prefix, sanitize_id};+use serde::{Deserialize, Serialize};use serde_json::{json, Value};use tracing::warn;@@ -411,6 +413,84 @@ fn parse_limit(params: &HashMap<String, String>) -> Result<i64, Box<Response>> {parse_bounded_int(params, "limit", DEFAULT_LIMIT, 1, MAX_LIMIT)}+#[derive(Serialize, Deserialize)]+struct CursorEnvelope {+ v: u8,+ endpoint: String,+ filters: String,+ state: Value,+}++fn invalid_cursor() -> Box<Response> {+ err_json_code(+ StatusCode::BAD_REQUEST,+ INVALID_QUERY_PARAM,+ "invalid 'cursor': expected a cursor returned by this endpoint with the same filters",+ )+}++fn encode_cursor(endpoint: &str, filters: &str, state: Value) -> String {+ URL_SAFE_NO_PAD.encode(+ serde_json::to_vec(&CursorEnvelope {+ v: 1,+ endpoint: endpoint.to_string(),+ filters: filters.to_string(),+ state,+ })+ .expect("cursor envelope serializes"),+ )+}++fn parse_cursor(+ params: &HashMap<String, String>,+ endpoint: &str,+ filters: &str,+) -> Result<Option<Value>, Box<Response>> {+ let Some(raw) = param(params, "cursor") else {+ return Ok(None);+ };+ if raw.len() > 4096 {+ return Err(invalid_cursor());+ }+ let envelope: CursorEnvelope = URL_SAFE_NO_PAD+ .decode(raw)+ .ok()+ .and_then(|bytes| serde_json::from_slice(&bytes).ok())+ .filter(|c: &CursorEnvelope| {+ c.v == 1 && c.endpoint == endpoint && c.filters == filters && c.state.is_object()+ })+ .ok_or_else(invalid_cursor)?;+ Ok(Some(envelope.state))+}++fn cursor_string(state: &Value, name: &str) -> Result<String, Box<Response>> {+ state+ .get(name)+ .and_then(Value::as_str)+ .map(str::to_string)+ .ok_or_else(invalid_cursor)+}++fn filter_signature(values: Value) -> String {+ serde_json::to_string(&values).expect("filter signature serializes")+}++fn cursor_window(state: &Value) -> Result<TimeWindow, Box<Response>> {+ let since = cursor_string(state, "since")?;+ let until = cursor_string(state, "until")?;+ let since = DateTime::parse_from_rfc3339(&since)+ .map(|t| t.with_timezone(&Utc))+ .map_err(|_| invalid_cursor())?;+ let until = DateTime::parse_from_rfc3339(&until)+ .map(|t| t.with_timezone(&Utc))+ .map_err(|_| invalid_cursor())?;+ Ok(TimeWindow {+ since,+ until,+ window_hours: ((until - since).num_milliseconds() as f64 / 3_600_000.0).max(0.0),+ })+}+fn rfc3339z(t: &DateTime<Utc>) -> String {t.to_rfc3339_opts(SecondsFormat::Millis, true)}@@ -492,13 +572,40 @@ struct KindQuery<'a> {until: String,limit: i64,newest_first: bool,+ cursor_t: Option<&'a str>,+ cursor_id: Option<&'a str>,+}++fn with_cursor_id(mut record: Value, id: String) -> Value {+ record["_cursor_id"] = Value::String(id);+ record+}++fn cursor_id_of(v: &Value) -> &str {+ v.get("_cursor_id").and_then(Value::as_str).unwrap_or("")+}++fn compare_records(a: &Value, b: &Value, newest_first: bool) -> std::cmp::Ordering {+ let by_time = match (+ DateTime::parse_from_rfc3339(t_of(a)),+ DateTime::parse_from_rfc3339(t_of(b)),+ ) {+ (Ok(a), Ok(b)) => a.cmp(&b),+ _ => t_of(a).cmp(t_of(b)),+ };+ let order = by_time.then_with(|| cursor_id_of(a).cmp(cursor_id_of(b)));+ if newest_first {+ order.reverse()+ } else {+ order+ }}async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>, sqlx::Error> {let fts = kq.q.and_then(fts_expression);let order = if kq.newest_first { "DESC" } else { "ASC" };let sql = format!(- r#"SELECT f.sync_id, f.machine_id, f.device_name, f.timestamp, f.app_name,+ r#"SELECT f.id, f.sync_id, f.machine_id, f.device_name, f.timestamp, f.app_name,f.window_name, f.browser_url, f.full_textFROM frames f{fts_join}@@ -506,10 +613,13 @@ async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<VaAND datetime(f.timestamp) >= datetime(?1) AND datetime(f.timestamp) <= datetime(?2)AND (?3 IS NULL OR f.machine_id = ?3)AND (?4 IS NULL OR f.app_name IS NULL OR lower(f.app_name) = lower(?4))- ORDER BY datetime(f.timestamp) {order}- LIMIT ?5"#,+ AND (?5 IS NULL OR datetime(f.timestamp) {cmp} datetime(?5)+ OR (datetime(f.timestamp) = datetime(?5) AND printf('frame:%020d', f.id) {cmp} ?6))+ ORDER BY datetime(f.timestamp) {order}, printf('frame:%020d', f.id) {order}+ LIMIT ?7"#,+ cmp = if kq.newest_first { "<" } else { ">" },fts_join = if fts.is_some() {- "JOIN frames_fts ON frames_fts.rowid = f.id AND frames_fts MATCH ?6"+ "JOIN frames_fts ON frames_fts.rowid = f.id AND frames_fts MATCH ?8"} else {""},@@ -517,6 +627,7 @@ async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Valet mut query = sqlx::query_as::<_,(+ i64,Option<String>,Option<String>,Option<String>,@@ -531,6 +642,8 @@ async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va.bind(&kq.until).bind(kq.device_id).bind(kq.app_name)+ .bind(kq.cursor_t)+ .bind(kq.cursor_id).bind(kq.limit);if let Some(f) = &fts {query = query.bind(f.clone());@@ -539,24 +652,27 @@ async fn query_frames(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<VaOk(rows.into_iter().map(- |(sync_id, machine_id, device_name, ts, app, window, url, text)| {- record_summary(- "frame",- Some(ts),- device_name,- machine_id,- app,- window,- url,- text,- None,- None,- None,- None,- None,- None,- local_id_from_sync_id(sync_id.as_deref()),- None,+ |(id, sync_id, machine_id, device_name, ts, app, window, url, text)| {+ with_cursor_id(+ record_summary(+ "frame",+ Some(ts),+ device_name,+ machine_id,+ app,+ window,+ url,+ text,+ None,+ None,+ None,+ None,+ None,+ None,+ local_id_from_sync_id(sync_id.as_deref()),+ None,+ ),+ format!("frame:{id:020}"),)},)@@ -567,17 +683,20 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Valet fts = kq.q.and_then(fts_expression);let order = if kq.newest_first { "DESC" } else { "ASC" };let sql = format!(- r#"SELECT p.device_id, p.device_label, p.timestamp, p.app_name,+ r#"SELECT p.rowid, p.device_id, p.device_label, p.timestamp, p.app_name,p.window_name, p.browser_url, p.text, p.frame_idFROM gateway_parsed_records p{fts_join}WHERE datetime(p.timestamp) >= datetime(?1) AND datetime(p.timestamp) <= datetime(?2)AND (?3 IS NULL OR p.device_id = ?3)AND (?4 IS NULL OR lower(p.app_name) = lower(?4))- ORDER BY datetime(p.timestamp) {order}- LIMIT ?5"#,+ AND (?5 IS NULL OR datetime(p.timestamp) {cmp} datetime(?5)+ OR (datetime(p.timestamp) = datetime(?5) AND printf('parsed:%020d', p.rowid) {cmp} ?6))+ ORDER BY datetime(p.timestamp) {order}, printf('parsed:%020d', p.rowid) {order}+ LIMIT ?7"#,+ cmp = if kq.newest_first { "<" } else { ">" },fts_join = if fts.is_some() {- "JOIN gateway_parsed_records_fts fts ON fts.rowid = p.rowid AND gateway_parsed_records_fts MATCH ?6"+ "JOIN gateway_parsed_records_fts fts ON fts.rowid = p.rowid AND gateway_parsed_records_fts MATCH ?8"} else {""},@@ -585,6 +704,7 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Valet mut query = sqlx::query_as::<_,(+ i64,String,String,String,@@ -599,6 +719,8 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Va.bind(&kq.until).bind(kq.device_id).bind(kq.app_name)+ .bind(kq.cursor_t)+ .bind(kq.cursor_id).bind(kq.limit);if let Some(f) = &fts {query = query.bind(f.clone());@@ -606,26 +728,31 @@ async fn query_parsed(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Valet rows = query.fetch_all(&db.pool).await?;Ok(rows.into_iter()- .map(|(device_id, label, ts, app, window, url, text, frame_id)| {- record_summary(- "parsed",- Some(ts),- Some(label),- Some(device_id),- Some(app),- Some(window),- url,- Some(text),- None,- None,- None,- None,- None,- None,- Some(frame_id),- None,- )- })+ .map(+ |(id, device_id, label, ts, app, window, url, text, frame_id)| {+ with_cursor_id(+ record_summary(+ "parsed",+ Some(ts),+ Some(label),+ Some(device_id),+ Some(app),+ Some(window),+ url,+ Some(text),+ None,+ None,+ None,+ None,+ None,+ None,+ Some(frame_id),+ None,+ ),+ format!("parsed:{id:020}"),+ )+ },+ ).collect())}@@ -633,7 +760,7 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Vallet fts = kq.q.and_then(fts_expression);let order = if kq.newest_first { "DESC" } else { "ASC" };let sql = format!(- r#"SELECT at.sync_id, ac.machine_id, gd.device_label, at.timestamp,+ r#"SELECT at.id, at.sync_id, ac.machine_id, gd.device_label, at.timestamp,at.transcription, s.nameFROM audio_transcriptions atJOIN audio_chunks ac ON at.audio_chunk_id = ac.id@@ -643,10 +770,13 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<ValWHERE ac.machine_id IS NOT NULLAND datetime(at.timestamp) >= datetime(?1) AND datetime(at.timestamp) <= datetime(?2)AND (?3 IS NULL OR ac.machine_id = ?3)- ORDER BY datetime(at.timestamp) {order}- LIMIT ?4"#,+ AND (?4 IS NULL OR datetime(at.timestamp) {cmp} datetime(?4)+ OR (datetime(at.timestamp) = datetime(?4) AND printf('audio:%020d', at.id) {cmp} ?5))+ ORDER BY datetime(at.timestamp) {order}, printf('audio:%020d', at.id) {order}+ LIMIT ?6"#,+ cmp = if kq.newest_first { "<" } else { ">" },fts_join = if fts.is_some() {- "JOIN audio_transcriptions_fts fts ON fts.rowid = at.id AND audio_transcriptions_fts MATCH ?5"+ "JOIN audio_transcriptions_fts fts ON fts.rowid = at.id AND audio_transcriptions_fts MATCH ?7"} else {""},@@ -654,6 +784,7 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Vallet mut query = sqlx::query_as::<_,(+ i64,Option<String>,Option<String>,Option<String>,@@ -665,6 +796,8 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Val.bind(&kq.since).bind(&kq.until).bind(kq.device_id)+ .bind(kq.cursor_t)+ .bind(kq.cursor_id).bind(kq.limit);if let Some(f) = &fts {query = query.bind(f.clone());@@ -673,24 +806,27 @@ async fn query_audio(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<ValOk(rows.into_iter().map(- |(_sync_id, machine_id, label, ts, transcription, speaker)| {- record_summary(- "audio",- Some(ts),- label,- machine_id,- None,- None,- None,- None,- Some(transcription),- speaker,- None,- None,- None,- None,- None,- None,+ |(id, _sync_id, machine_id, label, ts, transcription, speaker)| {+ with_cursor_id(+ record_summary(+ "audio",+ Some(ts),+ label,+ machine_id,+ None,+ None,+ None,+ None,+ Some(transcription),+ speaker,+ None,+ None,+ None,+ None,+ None,+ None,+ ),+ format!("audio:{id:020}"),)},)@@ -701,7 +837,7 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>let fts = kq.q.and_then(fts_expression);let order = if kq.newest_first { "DESC" } else { "ASC" };let sql = format!(- r#"SELECT ue.sync_id, ue.machine_id, gd.device_label, ue.timestamp,+ r#"SELECT ue.id, ue.sync_id, ue.machine_id, gd.device_label, ue.timestamp,ue.app_name, ue.browser_urlFROM ui_events ueLEFT JOIN gateway_devices gd ON gd.device_id = ue.machine_id@@ -710,10 +846,13 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>AND datetime(ue.timestamp) >= datetime(?1) AND datetime(ue.timestamp) <= datetime(?2)AND (?3 IS NULL OR ue.machine_id = ?3)AND (?4 IS NULL OR ue.app_name IS NULL OR lower(ue.app_name) = lower(?4))- ORDER BY datetime(ue.timestamp) {order}- LIMIT ?5"#,+ AND (?5 IS NULL OR datetime(ue.timestamp) {cmp} datetime(?5)+ OR (datetime(ue.timestamp) = datetime(?5) AND printf('ui:%020d', ue.id) {cmp} ?6))+ ORDER BY datetime(ue.timestamp) {order}, printf('ui:%020d', ue.id) {order}+ LIMIT ?7"#,+ cmp = if kq.newest_first { "<" } else { ">" },fts_join = if fts.is_some() {- "JOIN ui_events_fts ON ui_events_fts.rowid = ue.id AND ui_events_fts MATCH ?6"+ "JOIN ui_events_fts ON ui_events_fts.rowid = ue.id AND ui_events_fts MATCH ?8"} else {""},@@ -721,6 +860,7 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>let mut query = sqlx::query_as::<_,(+ i64,Option<String>,Option<String>,Option<String>,@@ -733,6 +873,8 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>.bind(&kq.until).bind(kq.device_id).bind(kq.app_name)+ .bind(kq.cursor_t)+ .bind(kq.cursor_id).bind(kq.limit);if let Some(f) = &fts {query = query.bind(f.clone());@@ -743,24 +885,27 @@ async fn query_ui(db: &DatabaseManager, kq: &KindQuery<'_>) -> Result<Vec<Value>// on the hosted side too.Ok(rows.into_iter()- .map(|(_sync_id, machine_id, label, ts, app, url)| {- record_summary(- "ui",- Some(ts),- label,- machine_id,- app,- None,- url,- None,- None,- None,- None,- None,- None,- None,- None,- None,+ .map(|(id, _sync_id, machine_id, label, ts, app, url)| {+ with_cursor_id(+ record_summary(+ "ui",+ Some(ts),+ label,+ machine_id,+ app,+ None,+ url,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ None,+ ),+ format!("ui:{id:020}"),)}).collect())@@ -773,7 +918,7 @@ async fn query_memories(let fts = kq.q.and_then(fts_expression);let order = if kq.newest_first { "DESC" } else { "ASC" };let sql = format!(- r#"SELECT m.sync_uuid, m.sync_modified_by, gd.device_label, m.created_at,+ r#"SELECT m.id, m.sync_uuid, m.sync_modified_by, gd.device_label, m.created_at,m.content, m.importance, m.tags, m.sourceFROM memories mLEFT JOIN gateway_devices gd ON gd.device_id = m.sync_modified_by@@ -781,10 +926,13 @@ async fn query_memories(WHERE m.sync_modified_by IS NOT NULLAND datetime(m.created_at) >= datetime(?1) AND datetime(m.created_at) <= datetime(?2)AND (?3 IS NULL OR m.sync_modified_by = ?3)- ORDER BY datetime(m.created_at) {order}- LIMIT ?4"#,+ AND (?4 IS NULL OR datetime(m.created_at) {cmp} datetime(?4)+ OR (datetime(m.created_at) = datetime(?4) AND printf('memory:%020d', m.id) {cmp} ?5))+ ORDER BY datetime(m.created_at) {order}, printf('memory:%020d', m.id) {order}+ LIMIT ?6"#,+ cmp = if kq.newest_first { "<" } else { ">" },fts_join = if fts.is_some() {- "JOIN memories_fts ON memories_fts.rowid = m.id AND memories_fts MATCH ?5"+ "JOIN memories_fts ON memories_fts.rowid = m.id AND memories_fts MATCH ?7"} else {""},@@ -792,6 +940,7 @@ async fn query_memories(let mut query = sqlx::query_as::<_,(+ i64,Option<String>,Option<String>,Option<String>,@@ -805,6 +954,8 @@ async fn query_memories(.bind(&kq.since).bind(&kq.until).bind(kq.device_id)+ .bind(kq.cursor_t)+ .bind(kq.cursor_id).bind(kq.limit);if let Some(f) = &fts {query = query.bind(f.clone());@@ -813,25 +964,28 @@ async fn query_memories(Ok(rows.into_iter().map(- |(sync_uuid, machine_id, label, created_at, content, importance, tags, source)| {+ |(id, sync_uuid, machine_id, label, created_at, content, importance, tags, source)| {let tags: Option<Vec<String>> = tags.and_then(|t| serde_json::from_str(&t).ok());- record_summary(- "memory",- Some(created_at),- label,- machine_id,- None,- None,- None,- None,- None,- None,- Some(content),- importance,- tags,- source,- None,- local_id_from_sync_id(sync_uuid.as_deref()),+ with_cursor_id(+ record_summary(+ "memory",+ Some(created_at),+ label,+ machine_id,+ None,+ None,+ None,+ None,+ None,+ None,+ Some(content),+ importance,+ tags,+ source,+ None,+ local_id_from_sync_id(sync_uuid.as_deref()),+ ),+ format!("memory:{id:020}"),)},)@@ -858,8 +1012,11 @@ async fn query_feedback(OR lower(COALESCE(f.comment, '')) LIKE '%' || lower(?4) || '%'OR lower(COALESCE(f.snapshot, '')) LIKE '%' || lower(?4) || '%'OR lower(f.context) LIKE '%' || lower(?4) || '%')- ORDER BY datetime(f.updated_at) {order}- LIMIT ?5"#,+ AND (?5 IS NULL OR datetime(f.updated_at) {cmp} datetime(?5)+ OR (datetime(f.updated_at) = datetime(?5) AND 'feedback:' || f.id {cmp} ?6))+ ORDER BY datetime(f.updated_at) {order}, 'feedback:' || f.id {order}+ LIMIT ?7"#,+ cmp = if kq.newest_first { "<" } else { ">" },);let rows = sqlx::query_as::<_,@@ -884,6 +1041,8 @@ async fn query_feedback(.bind(&kq.until).bind(kq.device_id).bind(kq.q)+ .bind(kq.cursor_t)+ .bind(kq.cursor_id).bind(kq.limit).fetch_all(&db.pool).await?;@@ -906,7 +1065,7 @@ async fn query_feedback(context,created_at,)| {- json!({+ with_cursor_id(json!({"kind": "feedback","t": updated_at,"device": device,@@ -938,7 +1097,7 @@ async fn query_feedback("context": serde_json::from_str::<Value>(&context).unwrap_or_else(|_| json!({})),"created_at": created_at,"updated_at": updated_at,- })+ }), format!("feedback:{id}"))},).collect())@@ -972,26 +1131,90 @@ fn t_of(v: &Value) -> &str {// ─── GET /api/enterprise/v1/devices ─────────────────────────────────────────-async fn devices(State(state): State<ApiState>) -> Response {- let rows: Result<Vec<(String, String, String, String)>, sqlx::Error> = sqlx::query_as(- "SELECT device_id, device_label, enrolled_at, last_seen \- FROM gateway_devices ORDER BY last_seen DESC",- )- .fetch_all(&state.db.pool)- .await;+async fn devices(+ State(state): State<ApiState>,+ Query(params): Query<HashMap<String, String>>,+) -> Response {+ let keyset = param(¶ms, "pagination") == Some("keyset");+ let limit = if keyset {+ match parse_limit(¶ms) {+ Ok(v) => v as usize,+ Err(resp) => return *resp,+ }+ } else {+ usize::MAX+ };+ let filters = "{}";+ let cursor = if keyset {+ match parse_cursor(¶ms, "devices", filters) {+ Ok(v) => v,+ Err(resp) => return *resp,+ }+ } else {+ None+ };+ let (as_of, after_device_id) = match cursor {+ Some(ref state) => match (+ cursor_string(state, "as_of"),+ cursor_string(state, "device_id"),+ ) {+ (Ok(as_of), Ok(device_id)) => (as_of, Some(device_id)),+ _ => return *invalid_cursor(),+ },+ None => (rfc3339z(&Utc::now()), None),+ };+ let rows: Result<Vec<(String, String, String, String)>, sqlx::Error> = if keyset {+ sqlx::query_as(+ "SELECT device_id, device_label, enrolled_at, last_seen \+ FROM gateway_devices \+ WHERE datetime(enrolled_at) <= datetime(?1) \+ AND (?2 IS NULL OR device_id > ?2) \+ ORDER BY device_id ASC LIMIT ?3",+ )+ .bind(&as_of)+ .bind(after_device_id.as_deref())+ .bind((limit + 1) as i64)+ .fetch_all(&state.db.pool)+ .await+ } else {+ sqlx::query_as(+ "SELECT device_id, device_label, enrolled_at, last_seen \+ FROM gateway_devices ORDER BY last_seen DESC",+ )+ .fetch_all(&state.db.pool)+ .await+ };match rows {- Ok(rows) => Json(json!({- "count": rows.len(),- "devices": rows.into_iter().map(|(device_id, label, enrolled_at, last_seen)| json!({- "device_id": device_id,- "label": label,- "platform": null,- "app_version": null,- "last_seen": last_seen,- "enrolled_at": enrolled_at,- })).collect::<Vec<_>>(),- }))- .into_response(),+ Ok(mut rows) => {+ let has_more = keyset && rows.len() > limit;+ if keyset {+ rows.truncate(limit);+ }+ let next_cursor = has_more && !rows.is_empty();+ let next_cursor = next_cursor.then(|| {+ encode_cursor(+ "devices",+ filters,+ json!({ "as_of": as_of, "device_id": rows.last().unwrap().0 }),+ )+ });+ let mut body = json!({+ "count": rows.len(),+ "devices": rows.into_iter().map(|(device_id, label, enrolled_at, last_seen)| json!({+ "device_id": device_id,+ "label": label,+ "member_email": null,+ "platform": null,+ "app_version": null,+ "last_seen": last_seen,+ "enrolled_at": enrolled_at,+ })).collect::<Vec<_>>(),+ });+ if keyset {+ body["next_cursor"] = json!(next_cursor);+ }+ Json(body).into_response()+ }Err(e) => err_json(StatusCode::INTERNAL_SERVER_ERROR,&format!("device list failed: {e}"),@@ -1005,26 +1228,57 @@ async fn search(State(state): State<ApiState>,Query(params): Query<HashMap<String, String>>,) -> Response {+ let keyset = param(¶ms, "pagination") == Some("keyset");let q = params.get("q").map(|s| s.trim()).unwrap_or("");let device_id = param(¶ms, "device_id");let app_name = param(¶ms, "app_name");- let window = match parse_time_window(¶ms) {- Ok(w) => w,- Err(resp) => return *resp,+ let filters = filter_signature(json!({+ "q": q,+ "device_id": device_id,+ "app_name": app_name,+ "since": param(¶ms, "since"),+ "until": param(¶ms, "until"),+ "since_hours_ago": param(¶ms, "since_hours_ago"),+ }));+ let cursor = if keyset {+ match parse_cursor(¶ms, "search", &filters) {+ Ok(v) => v,+ Err(resp) => return *resp,+ }+ } else {+ None+ };+ let window = match cursor.as_ref() {+ Some(state) => match cursor_window(state) {+ Ok(w) => w,+ Err(resp) => return *resp,+ },+ None => match parse_time_window(¶ms) {+ Ok(w) => w,+ Err(resp) => return *resp,+ },+ };+ let cursor_position = match cursor.as_ref() {+ Some(state) => match (cursor_string(state, "t"), cursor_string(state, "id")) {+ (Ok(t), Ok(id)) => Some((t, id)),+ _ => return *invalid_cursor(),+ },+ None => None,};let limit = match parse_limit(¶ms) {Ok(l) => l,Err(resp) => return *resp,};-let kq = KindQuery {q: if q.is_empty() { None } else { Some(q) },device_id,app_name,since: rfc3339z(&window.since),until: rfc3339z(&window.until),- limit,+ limit: if keyset { limit + 1 } else { limit },newest_first: true,+ cursor_t: cursor_position.as_ref().map(|p| p.0.as_str()),+ cursor_id: cursor_position.as_ref().map(|p| p.1.as_str()),};let mut results = match query_kinds(&state.db,@@ -1037,10 +1291,33 @@ async fn search(Err(e) => return err_json(StatusCode::INTERNAL_SERVER_ERROR, &format!("search: {e}")),};let records_scanned = results.len();- results.sort_by(|a, b| t_of(b).cmp(t_of(a)));+ results.sort_by(|a, b| {+ if keyset {+ compare_records(a, b, true)+ } else {+ t_of(b).cmp(t_of(a))+ }+ });+ let has_more = keyset && results.len() > limit as usize;results.truncate(limit as usize);+ let next_cursor = (has_more && !results.is_empty()).then(|| {+ let last = results.last().unwrap();+ encode_cursor(+ "search",+ &filters,+ json!({+ "since": rfc3339z(&window.since),+ "until": rfc3339z(&window.until),+ "t": t_of(last),+ "id": cursor_id_of(last),+ }),+ )+ });+ for result in &mut results {+ result.as_object_mut().map(|o| o.remove("_cursor_id"));+ }- Json(json!({+ let mut body = json!({"query": q,"device_id": device_id,"app_name": app_name,@@ -1051,8 +1328,11 @@ async fn search("truncated": false,"result_count": results.len(),"results": results,- }))- .into_response()+ });+ if keyset {+ body["next_cursor"] = json!(next_cursor);+ }+ Json(body).into_response()}// ─── GET /api/enterprise/v1/records ─────────────────────────────────────────@@ -1061,19 +1341,47 @@ async fn records(State(state): State<ApiState>,Query(params): Query<HashMap<String, String>>,) -> Response {+ let keyset = param(¶ms, "pagination") == Some("keyset");let device_id = param(¶ms, "device_id");let kind_filter = param(¶ms, "kind").map(|s| s.to_ascii_lowercase()).unwrap_or_else(|| "all".to_string());- let window = match parse_time_window(¶ms) {- Ok(w) => w,- Err(resp) => return *resp,+ let filters = filter_signature(json!({+ "device_id": device_id,+ "kind": kind_filter,+ "since": param(¶ms, "since"),+ "until": param(¶ms, "until"),+ "since_hours_ago": param(¶ms, "since_hours_ago"),+ }));+ let cursor = if keyset {+ match parse_cursor(¶ms, "records", &filters) {+ Ok(v) => v,+ Err(resp) => return *resp,+ }+ } else {+ None+ };+ let window = match cursor.as_ref() {+ Some(state) => match cursor_window(state) {+ Ok(w) => w,+ Err(resp) => return *resp,+ },+ None => match parse_time_window(¶ms) {+ Ok(w) => w,+ Err(resp) => return *resp,+ },+ };+ let cursor_position = match cursor.as_ref() {+ Some(state) => match (cursor_string(state, "t"), cursor_string(state, "id")) {+ (Ok(t), Ok(id)) => Some((t, id)),+ _ => return *invalid_cursor(),+ },+ None => None,};let limit = match parse_limit(¶ms) {Ok(l) => l,Err(resp) => return *resp,};-let kinds: Vec<&str> = match kind_filter.as_str() {"all" => vec!["frame", "parsed", "audio", "ui", "memory", "feedback"],k => vec![k],@@ -1084,18 +1392,42 @@ async fn records(app_name: None,since: rfc3339z(&window.since),until: rfc3339z(&window.until),- limit,+ limit: if keyset { limit + 1 } else { limit },newest_first: false,+ cursor_t: cursor_position.as_ref().map(|p| p.0.as_str()),+ cursor_id: cursor_position.as_ref().map(|p| p.1.as_str()),};let mut recs = match query_kinds(&state.db, &kinds, &kq).await {Ok(r) => r,Err(e) => return err_json(StatusCode::INTERNAL_SERVER_ERROR, &format!("records: {e}")),};- // Hosted sorts the collected page ascending by `t` (lexicographic).- recs.sort_by(|a, b| t_of(a).cmp(t_of(b)));+ recs.sort_by(|a, b| {+ if keyset {+ compare_records(a, b, false)+ } else {+ t_of(a).cmp(t_of(b))+ }+ });+ let has_more = keyset && recs.len() > limit as usize;recs.truncate(limit as usize);+ let next_cursor = (has_more && !recs.is_empty()).then(|| {+ let last = recs.last().unwrap();+ encode_cursor(+ "records",+ &filters,+ json!({+ "since": rfc3339z(&window.since),+ "until": rfc3339z(&window.until),+ "t": t_of(last),+ "id": cursor_id_of(last),+ }),+ )+ });+ for record in &mut recs {+ record.as_object_mut().map(|o| o.remove("_cursor_id"));+ }- Json(json!({+ let mut body = json!({"device_id": device_id,"kind": kind_filter,"window_hours": window.window_hours,@@ -1104,8 +1436,11 @@ async fn records("truncated": false,"record_count": recs.len(),"records": recs,- }))- .into_response()+ });+ if keyset {+ body["next_cursor"] = json!(next_cursor);+ }+ Json(body).into_response()}// ─── GET /api/enterprise/v1/files ───────────────────────────────────────────@@ -1115,22 +1450,42 @@ async fn files(Query(params): Query<HashMap<String, String>>,) -> Response {let device_id = param(¶ms, "device_id");- let window = match parse_time_window(¶ms) {- Ok(w) => w,- Err(resp) => return *resp,+ let filters = filter_signature(json!({+ "device_id": device_id,+ "since": param(¶ms, "since"),+ "until": param(¶ms, "until"),+ "since_hours_ago": param(¶ms, "since_hours_ago"),+ }));+ let legacy_offset = param(¶ms, "cursor").and_then(|raw| raw.parse::<usize>().ok());+ let cursor = if legacy_offset.is_none() {+ match parse_cursor(¶ms, "files", &filters) {+ Ok(v) => v,+ Err(resp) => return *resp,+ }+ } else {+ None+ };+ let window = match cursor.as_ref() {+ Some(state) => match cursor_window(state) {+ Ok(w) => w,+ Err(resp) => return *resp,+ },+ None => match parse_time_window(¶ms) {+ Ok(w) => w,+ Err(resp) => return *resp,+ },+ };+ let after_key = match cursor.as_ref() {+ Some(state) => match cursor_string(state, "key") {+ Ok(key) => Some(key),+ Err(resp) => return *resp,+ },+ None => None,};let page_size = match parse_bounded_int(¶ms, "page_size", 100, 1, 1000) {Ok(v) => v as usize,Err(resp) => return *resp,};- // `cursor` is an OPAQUE token by contract (the hosted target's is a storage- // continuation token, not a number), so an unparseable one is deliberately- // page-one rather than a 400 — a client is not supposed to construct it.- let offset: usize = params- .get("cursor")- .and_then(|s| s.parse().ok())- .unwrap_or(0);-let prefix = match device_id {Some(d) => format!("{}{}/",@@ -1145,21 +1500,32 @@ async fn files(};let since = rfc3339z(&window.since);let until = rfc3339z(&window.until);- let in_window: Vec<_> = listed+ let mut in_window: Vec<_> = listed.entries.into_iter().filter(|e| {- e.last_modified- .as_deref()- .map(|lm| lm >= since.as_str() && lm <= until.as_str())- .unwrap_or(true)+ after_key+ .as_ref()+ .is_none_or(|key| e.key.as_str() > key.as_str())+ && e.last_modified+ .as_deref()+ .map(|lm| lm >= since.as_str() && lm <= until.as_str())+ .unwrap_or(true)}).collect();+ in_window.sort_by(|a, b| a.key.cmp(&b.key));let total = in_window.len();+ let has_more = legacy_offset.map_or(total > page_size, |offset| {+ offset.saturating_add(page_size) < total+ });+ if let Some(offset) = legacy_offset {+ in_window = in_window.into_iter().skip(offset).take(page_size).collect();+ } else {+ in_window.truncate(page_size);+ }+ let last_key = in_window.last().map(|entry| entry.key.clone());let page: Vec<Value> = in_window.into_iter()- .skip(offset)- .take(page_size).map(|e| {let dev = e.key@@ -1176,8 +1542,19 @@ async fn files(})}).collect();- let next_cursor = if offset + page.len() < total {- Some((offset + page.len()).to_string())+ let next_cursor = if has_more && !page.is_empty() {+ Some(match legacy_offset {+ Some(offset) => offset.saturating_add(page.len()).to_string(),+ None => encode_cursor(+ "files",+ &filters,+ json!({+ "since": rfc3339z(&window.since),+ "until": rfc3339z(&window.until),+ "key": last_key.unwrap(),+ }),+ ),+ })} else {None};@@ -1288,33 +1665,66 @@ async fn rollups(State(state): State<ApiState>,Query(params): Query<HashMap<String, String>>,) -> Response {- let device: String = param(¶ms, "device")- .map(|s| {- s.chars()- .map(|c| {- if c.is_ascii_alphanumeric() || " _.-".contains(c) {- c- } else {- '_'- }- })- .collect()- })- .filter(|s: &String| !s.is_empty())- .unwrap_or_else(|| "org".to_string());- let today = Utc::now().format("%Y-%m-%d").to_string();- let to = param(¶ms, "to").map(str::to_string).unwrap_or(today);- let from = param(¶ms, "from")- .map(str::to_string)- .unwrap_or_else(|| {- (Utc::now() - ChronoDuration::days(31))- .format("%Y-%m-%d")- .to_string()- });+ let keyset = param(¶ms, "pagination") == Some("keyset");let limit = match parse_bounded_int(¶ms, "limit", 35, 1, 100) {Ok(v) => v as usize,Err(resp) => return *resp,};+ let filters = filter_signature(json!({+ "device": param(¶ms, "device"),+ "from": param(¶ms, "from"),+ "to": param(¶ms, "to"),+ }));+ let cursor = if keyset {+ match parse_cursor(¶ms, "rollups", &filters) {+ Ok(v) => v,+ Err(resp) => return *resp,+ }+ } else {+ None+ };+ let (device, from, to, as_of, after) = match cursor {+ Some(ref state) => match (+ cursor_string(state, "device"),+ cursor_string(state, "from"),+ cursor_string(state, "to"),+ cursor_string(state, "as_of"),+ cursor_string(state, "day"),+ cursor_string(state, "key"),+ ) {+ (Ok(device), Ok(from), Ok(to), Ok(as_of), Ok(day), Ok(key)) => {+ (device, from, to, as_of, Some((day, key)))+ }+ _ => return *invalid_cursor(),+ },+ None => {+ let device: String = param(¶ms, "device")+ .map(|s| {+ s.chars()+ .map(|c| {+ if c.is_ascii_alphanumeric() || " _.-".contains(c) {+ c+ } else {+ '_'+ }+ })+ .collect()+ })+ .filter(|s: &String| !s.is_empty())+ .unwrap_or_else(|| "org".to_string());+ let to = param(¶ms, "to")+ .map(str::to_string)+ .unwrap_or_else(|| Utc::now().format("%Y-%m-%d").to_string());+ let from = param(¶ms, "from")+ .map(str::to_string)+ .unwrap_or_else(|| {+ (Utc::now() - ChronoDuration::days(31))+ .format("%Y-%m-%d")+ .to_string()+ });+ (device, from, to, rfc3339z(&Utc::now()), None)+ }+ };let prefix = format!("rollups/{}/{}/", sanitize_id(&state.license_id, 64), device);let listed = match state.source.list(&ListRequest::new(&prefix)).await {@@ -1325,18 +1735,32 @@ async fn rollups(.entries.into_iter().filter_map(|e| {+ if keyset+ && e.last_modified+ .as_deref()+ .is_some_and(|modified| modified > as_of.as_str())+ {+ return None;+ }let day = e.key.strip_prefix(&prefix)?.strip_suffix(".json")?;let ok = day.len() == 10&& day.chars().enumerate().all(|(i, c)| match i {4 | 7 => c == '-',_ => c.is_ascii_digit(),});- (ok && day >= from.as_str() && day <= to.as_str())+ let after_cursor = after.as_ref().is_none_or(|(after_day, after_key)| {+ day < after_day.as_str()+ || (day == after_day && e.key.as_str() < after_key.as_str())+ });+ (ok && day >= from.as_str() && day <= to.as_str() && after_cursor).then(|| (day.to_string(), e.key.clone()))}).collect();- days.sort_by(|a, b| b.0.cmp(&a.0));+ days.sort_by(|a, b| b.0.cmp(&a.0).then_with(|| b.1.cmp(&a.1)));+ let has_more = keyset && days.len() > limit;days.truncate(limit);+ let selected_count = days.len();+ let last_position = days.last().cloned();let mut out = Vec::with_capacity(days.len());for (day, key) in days {@@ -1349,15 +1773,33 @@ async fn rollups(}}- Json(json!({+ let next_cursor = (has_more && selected_count > 0).then(|| {+ let (day, key) = last_position.unwrap();+ encode_cursor(+ "rollups",+ &filters,+ json!({+ "device": &device,+ "from": &from,+ "to": &to,+ "as_of": &as_of,+ "day": day,+ "key": key,+ }),+ )+ });+ let mut body = json!({"ok": true,"device": device,"from": from,"to": to,"count": out.len(),"rollups": out,- }))- .into_response()+ });+ if keyset {+ body["next_cursor"] = json!(next_cursor);+ }+ Json(body).into_response()}#[cfg(test)]@@ -1535,6 +1977,22 @@ mod tests {}}+ #[test]+ fn keyset_cursors_are_endpoint_and_filter_bound() {+ let p = |v: &str| HashMap::from([("cursor".to_string(), v.to_string())]);+ assert!(parse_cursor(&HashMap::new(), "search", "filters")+ .unwrap()+ .is_none());+ let token = encode_cursor("search", "filters", json!({ "t": "x", "id": "y" }));+ assert_eq!(+ parse_cursor(&p(&token), "search", "filters").unwrap(),+ Some(json!({ "t": "x", "id": "y" }))+ );+ assert!(parse_cursor(&p(&token), "records", "filters").is_err());+ assert!(parse_cursor(&p(&token), "search", "different").is_err());+ assert!(parse_cursor(&p("not-a-cursor"), "search", "filters").is_err());+ }+/// The window parser refuses at the ROUTE, with the machine code the hosted/// side emits — not just in the unit above.#[tokio::test]@@ -1703,6 +2161,13 @@ mod tests {dir: &tempfile::TempDir,policy: Option<crate::auth::PolicyStore>,) -> Router {+ seeded_router_and_db_with_policy(dir, policy).await.0+ }++ async fn seeded_router_and_db_with_policy(+ dir: &tempfile::TempDir,+ policy: Option<crate::auth::PolicyStore>,+ ) -> (Router, Arc<DatabaseManager>) {let db = Arc::new(DatabaseManager::new(dir.path().join("gateway.db").to_str().unwrap(),@@ -1771,6 +2236,14 @@ mod tests {.await.unwrap();}+ for day in ["2026-07-21", "2026-07-22"] {+ src.put_for_tests(+ &format!("rollups/lic-1/org/{day}.json"),+ serde_json::to_vec(&json!({ "day": day, "records": 1 })).unwrap(),+ )+ .await+ .unwrap();+ }let ingestor = Ingestor::new(src.clone() as Arc<dyn BlobSource>,db.clone(),@@ -1780,7 +2253,8 @@ mod tests {.await.unwrap();ingestor.run_once().await.unwrap();- router(db, src, "lic-1".to_string(), policy)+ let app = router(db.clone(), src, "lic-1".to_string(), policy);+ (app, db)}async fn get_json(router: &Router, uri: &str) -> (StatusCode, Value) {@@ -1806,6 +2280,7 @@ mod tests {).await;assert_eq!(status, StatusCode::OK);+ assert!(body.get("next_cursor").is_none());let results = body["results"].as_array().unwrap();// 2 frames + 2 audio + 2 memories match "roadmap"-adjacent tokens?// "roadmap" appears in frame text, audio transcription, and memory@@ -1848,6 +2323,22 @@ mod tests {let mut sorted = ts.clone();sorted.sort_by(|a, b| b.cmp(a));assert_eq!(ts, sorted);++ let (_, first) = get_json(+ &router,+ "/api/enterprise/v1/search?q=roadmap&since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1",+ )+ .await;+ assert_eq!(first["result_count"], 1);+ let cursor = first["next_cursor"].as_str().unwrap();+ assert!(!cursor.bytes().all(|b| b.is_ascii_digit()));+ let (_, second) = get_json(+ &router,+ &format!("/api/enterprise/v1/search?q=roadmap&since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1&cursor={cursor}"),+ )+ .await;+ assert_eq!(second["result_count"], 1);+ assert_ne!(first["results"][0], second["results"][0]);}#[tokio::test]@@ -1883,6 +2374,7 @@ mod tests {).await;assert_eq!(status, StatusCode::OK);+ assert!(body.get("next_cursor").is_none());let records = body["records"].as_array().unwrap();assert_eq!(records.len(), 2);assert!(records.iter().all(|r| r["kind"] == "memory"));@@ -1894,6 +2386,21 @@ mod tests {sorted.sort();assert_eq!(ts, sorted, "records are ascending by t");assert_eq!(records[0]["memory_id"], 1);++ let (_, first) = get_json(+ &router,+ "/api/enterprise/v1/records?since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1",+ )+ .await;+ assert_eq!(first["record_count"], 1);+ let cursor = first["next_cursor"].as_str().unwrap();+ let (_, second) = get_json(+ &router,+ &format!("/api/enterprise/v1/records?since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1&cursor={cursor}"),+ )+ .await;+ assert_eq!(second["record_count"], 1);+ assert_ne!(first["records"][0], second["records"][0]);}#[tokio::test]@@ -1935,11 +2442,126 @@ mod tests {assert_eq!(status, StatusCode::OK);assert_eq!(body["count"], 2);let devices = body["devices"].as_array().unwrap();- // Ordered by last_seen DESC — dev-b's records are newer.assert_eq!(devices[0]["device_id"], "dev-b");assert_eq!(devices[0]["label"], "bob-mbp");+ assert!(devices[0]["member_email"].is_null());assert!(devices[0]["platform"].is_null());assert!(devices[0].get("last_seen").is_some());+ assert!(body.get("next_cursor").is_none());++ let (_, first) = get_json(+ &router,+ "/api/enterprise/v1/devices?pagination=keyset&limit=1",+ )+ .await;+ assert_eq!(first["count"], 1);+ assert_eq!(first["devices"][0]["device_id"], "dev-a");+ let cursor = first["next_cursor"].as_str().unwrap();+ let (_, second) = get_json(+ &router,+ &format!("/api/enterprise/v1/devices?pagination=keyset&limit=1&cursor={cursor}"),+ )+ .await;+ assert_eq!(second["count"], 1);+ assert_ne!(first["devices"][0], second["devices"][0]);+ }++ #[tokio::test]+ async fn keyset_pages_do_not_shift_when_rows_change_between_requests() {+ let dir = tempfile::tempdir().unwrap();+ let router = seeded_router(&dir).await;+ let (_, first) = get_json(+ &router,+ "/api/enterprise/v1/devices?pagination=keyset&limit=1",+ )+ .await;+ assert_eq!(first["devices"][0]["device_id"], "dev-a");+ let cursor = first["next_cursor"].as_str().unwrap();+ let writable = sqlx::sqlite::SqlitePoolOptions::new()+ .max_connections(1)+ .connect_with(+ sqlx::sqlite::SqliteConnectOptions::new()+ .filename(dir.path().join("gateway.db"))+ .read_only(false),+ )+ .await+ .unwrap();++ sqlx::query(+ "INSERT INTO gateway_devices (device_id, device_label, enrolled_at, last_seen) \+ VALUES ('dev-0', 'new-before-cursor', '2026-07-22T08:00:00Z', '2026-07-22T12:00:00Z')",+ )+ .execute(&writable)+ .await+ .unwrap();++ let (_, second) = get_json(+ &router,+ &format!("/api/enterprise/v1/devices?pagination=keyset&limit=1&cursor={cursor}"),+ )+ .await;+ assert_eq!(+ second["devices"][0]["device_id"], "dev-b",+ "an inserted row before the keyset must not repeat or shift dev-a"+ );++ let (_, first) = get_json(+ &router,+ "/api/enterprise/v1/records?since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1",+ )+ .await;+ let first_record = first["records"][0].clone();+ let cursor = first["next_cursor"].as_str().unwrap();+ sqlx::query(+ "UPDATE memories SET created_at = '2026-07-22T09:00:00Z' \+ WHERE sync_modified_by = 'dev-b'",+ )+ .execute(&writable)+ .await+ .unwrap();+ let (_, second) = get_json(+ &router,+ &format!("/api/enterprise/v1/records?since=2026-07-22T00:00:00Z&until=2026-07-23T00:00:00Z&pagination=keyset&limit=1&cursor={cursor}"),+ )+ .await;+ assert_ne!(+ second["records"][0], first_record,+ "moving a later row before the keyset must not duplicate page one"+ );+ }++ #[tokio::test]+ async fn rollups_return_continuation_cursors() {+ let dir = tempfile::tempdir().unwrap();+ let router = seeded_router(&dir).await;++ let (_, legacy) = get_json(+ &router,+ "/api/enterprise/v1/rollups?from=2026-07-01&to=2026-07-31",+ )+ .await;+ assert!(legacy.get("next_cursor").is_none());++ let (status, first) = get_json(+ &router,+ "/api/enterprise/v1/rollups?from=2026-07-01&to=2026-07-31&pagination=keyset&limit=1",+ )+ .await;+ assert_eq!(status, StatusCode::OK);+ assert_eq!(first["count"], 1);+ assert_eq!(first["rollups"][0]["day"], "2026-07-22");+ let cursor = first["next_cursor"].as_str().unwrap();++ let (_, second) = get_json(+ &router,+ &format!(+ "/api/enterprise/v1/rollups?from=2026-07-01&to=2026-07-31&pagination=keyset&limit=1&cursor={cursor}"+ ),+ )+ .await;+ assert_eq!(second["count"], 1);+ assert_eq!(second["rollups"][0]["day"], "2026-07-21");+ assert!(second["next_cursor"].is_null());}#[tokio::test]@@ -1957,6 +2579,32 @@ mod tests {let key = body["files"][0]["key"].as_str().unwrap().to_string();assert!(key.starts_with("enterprise-telemetry/lic-1/"));+ let (_, first) = get_json(+ &router,+ "/api/enterprise/v1/files?since=2020-01-01T00:00:00Z&page_size=1",+ )+ .await;+ let cursor = first["next_cursor"].as_str().unwrap();+ assert!(!cursor.bytes().all(|b| b.is_ascii_digit()));+ let (_, second) = get_json(+ &router,+ &format!(+ "/api/enterprise/v1/files?since=2020-01-01T00:00:00Z&page_size=1&cursor={cursor}"+ ),+ )+ .await;+ assert_eq!(second["count"], 1);+ assert_ne!(first["files"][0], second["files"][0]);++ let (_, legacy_second) = get_json(+ &router,+ "/api/enterprise/v1/files?since=2020-01-01T00:00:00Z&page_size=1&cursor=1",+ )+ .await;+ assert_eq!(legacy_second["count"], 1);+ assert_eq!(legacy_second["files"][0], second["files"][0]);+ assert!(legacy_second["next_cursor"].is_null());+let resp = router.clone().oneshot(