Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
perf(web): batch access checks for Caps grid analytics
Aditya Garud87ffd7ed7734
diff --git a/apps/web/__tests__/unit/caps-analytics-round-trips.test.ts b/apps/web/__tests__/unit/caps-analytics-round-trips.test.tsnew file mode 100644index 000000000..036a2b382--- /dev/null+++ b/apps/web/__tests__/unit/caps-analytics-round-trips.test.ts@@ -0,0 +1,359 @@+import { CurrentUser, Organisation, User, Video } from "@cap/web-domain";+import { Cause, Effect, Exit, Logger, LogLevel, Option } from "effect";+import { beforeEach, describe, expect, it, vi } from "vitest";++type AccessFixture = {+ id: string;+ public?: boolean;+ owner?: boolean;+ orgMember?: boolean;+ spaceMember?: boolean;+ password?: string;+ spacePasswords?: string[];+ domain?: string;+ grant?: boolean;+};+const mocks = vi.hoisted(() => ({+ query: vi.fn(),+ latency: 0,+ size: 50,+ owner: false,+ fixtures: null as AccessFixture[] | null,+ failTable: "",+}));+vi.mock("@cap/database", async () => {+ const { drizzle } = await import("drizzle-orm/mysql-proxy");+ return { db: () => drizzle(mocks.query) };+});+vi.mock("@cap/utils", async () => {+ const { isEmailAllowedByRestriction } = await import(+ "../../../../packages/utils/src/helpers"+ );+ return { dub: {}, isEmailAllowedByRestriction };+});+vi.mock("@cap/web-backend/src/Storage/index", async () => {+ const { Effect } = await import("effect");+ class Storage extends Effect.Service<Storage>()("Storage", {+ sync: () => ({}),+ }) {}+ return { Storage };+});+vi.mock("@cap/web-backend/src/Tinybird/index", async () => {+ const { Effect } = await import("effect");+ class Tinybird extends Effect.Service<Tinybird>()("Tinybird", {+ succeed: {+ querySql: () =>+ Effect.succeed({+ data: Array.from({ length: mocks.size }, (_, index) => ({+ pathname: `/s/video-${index}`,+ views: index + 1,+ })),+ }),+ },+ }) {}+ return { Tinybird };+});++const { Videos } = await import("@cap/web-backend/src/Videos/index");+const { VideosPolicy } = await import(+ "@cap/web-backend/src/Videos/VideosPolicy"+);+const { VideosRepo } = await import("@cap/web-backend/src/Videos/VideosRepo");+const { Database } = await import("@cap/web-backend/src/Database");+const { Storage } = await import("@cap/web-backend/src/Storage/index");+const { Tinybird } = await import("@cap/web-backend/src/Tinybird/index");++const user: CurrentUser["Type"] = {+ id: User.UserId.make("viewer"),+ email: "[email protected]",+ activeOrganizationId: Organisation.OrganisationId.make("org"),+ iconUrlOrKey: Option.none(),+};+function ids() {+ return Array.from({ length: mocks.size }, (_, index) =>+ Video.VideoId.make(`video-${index}`),+ );+}+function run() {+ return Effect.flatMap(Videos, (videos) =>+ videos.getAnalyticsBulk(ids()),+ ).pipe(+ Effect.provide(Videos.DefaultWithoutDependencies),+ Effect.provide([+ VideosPolicy.Default,+ VideosRepo.Default,+ Storage.Default,+ Tinybird.Default,+ Database.Default,+ ]),+ Effect.provideService(CurrentUser, user),+ Logger.withMinimumLogLevel(LogLevel.None),+ Effect.runPromise,+ );+}+function reply(sql: string, params: unknown[] = []) {+ if (mocks.failTable && sql.includes(`from \`${mocks.failTable}\``))+ throw new Error("Database unavailable");+ if (mocks.fixtures) {+ const selected = mocks.fixtures.filter((fixture) =>+ params.includes(fixture.id),+ );+ if (sql.includes("from `organization_members`")) {+ expect(params[0]).toBe("viewer");+ return selected+ .filter((fixture) => fixture.orgMember)+ .map((fixture) =>+ sql.includes("`shared_videos`.`videoId`,")+ ? [fixture.id, "org-membership"]+ : ["org-membership"],+ );+ }+ if (sql.includes("from `space_members`")) {+ expect(params[0]).toBe("viewer");+ return selected+ .filter((fixture) => fixture.spaceMember)+ .map((fixture) =>+ sql.includes("`space_videos`.`videoId`,")+ ? [fixture.id, "space-membership"]+ : ["space-membership"],+ );+ }+ if (sql.includes("from `space_videos`"))+ return selected.flatMap((fixture) =>+ (fixture.spacePasswords ?? []).map((password) =>+ sql.startsWith("select `space_videos`.`videoId`")+ ? [fixture.id, password]+ : ["space", "Space", password],+ ),+ );+ if (sql.includes("from `organizations`"))+ return mocks.fixtures+ .filter((fixture) => params.includes(`org-${fixture.id}`))+ .map((fixture) => [fixture.domain ?? null]);+ if (sql.includes("from `video_viewer_grants`"))+ return selected.filter((fixture) => fixture.grant).map(() => ["grant"]);+ throw new Error(`Unexpected policy query: ${sql}`);+ }+ if (sql.includes("from `videos`"))+ return ids().map((id) => [+ id,+ "org",+ mocks.owner ? "viewer" : "owner",+ 0,+ null,+ ]);+ if (sql.includes("from `organization_members`"))+ return sql.includes("`shared_videos`.`videoId`,")+ ? ids().map((id) => [id, "membership"])+ : [["membership"]];+ if (+ sql.includes("from `space_members`") ||+ sql.includes("from `space_videos`")+ )+ return [];+ throw new Error(`Unexpected query: ${sql}`);+}+async function countRoundTrips() {+ const pending: Array<() => void> = [];+ mocks.query.mockImplementation(+ (sql: string, params: unknown[]) =>+ new Promise((resolve) =>+ pending.push(() => resolve({ rows: reply(sql, params) })),+ ),+ );+ let settled = false;+ const result = run().finally(() => {+ settled = true;+ });+ let rounds = 0;+ let idleTicks = 0;+ while (!settled) {+ const issued = mocks.query.mock.calls.length;+ await new Promise((resolve) => setTimeout(resolve, 0));+ if (mocks.query.mock.calls.length !== issued || pending.length === 0) {+ idleTicks = 0;+ continue;+ }+ if (++idleTicks < 3) continue;+ idleTicks = 0;+ rounds++;+ for (const release of pending.splice(0)) release();+ }+ await result;+ return rounds;+}+function timedQueries() {+ mocks.query.mockImplementation(async (sql: string, params: unknown[]) => {+ if (mocks.latency)+ await new Promise((resolve) => setTimeout(resolve, mocks.latency));+ return { rows: reply(sql, params) };+ });+}+beforeEach(() => {+ mocks.size = 50;+ mocks.fixtures = null;+ mocks.failTable = "";+ mocks.owner = false;+ mocks.latency = 0;+ timedQueries();+});+describe("Caps grid analytics database load", () => {+ it.each([50, 200])("returns every shared count for %i Caps", async (size) => {+ mocks.size = size;+ const results = await run();+ expect(results.map((result) => Exit.getOrElse(result, () => null))).toEqual(+ ids().map((_, index) => ({ count: index + 1 })),+ );+ expect(mocks.query).toHaveBeenCalledTimes(4);+ });+ it("keeps owner analytics to one query", async () => {+ mocks.owner = true;+ await run();+ expect(mocks.query).toHaveBeenCalledTimes(1);+ });+ it.skipIf(!process.env.CAP_BENCH_CAPS_ANALYTICS)(+ "measures 20 ms database latency for My Caps and Shared Caps",+ async () => {+ for (const owner of [true, false]) {+ for (const size of [15, 50, 200]) {+ mocks.owner = owner;+ mocks.size = size;+ mocks.latency = 20;+ const rounds = await countRoundTrips();+ timedQueries();+ const times: number[] = [];+ let queries = 0;+ for (let iteration = 0; iteration < 7; iteration++) {+ mocks.query.mockClear();+ const start = performance.now();+ const results = await run();+ times.push(performance.now() - start);+ queries = mocks.query.mock.calls.length;+ expect(results.every(Exit.isSuccess)).toBe(true);+ }+ times.sort((a, b) => a - b);+ console.log(+ JSON.stringify({+ owner,+ size,+ queries,+ rounds,+ medianMs: times[3],+ }),+ );+ }+ }+ },+ );+});++const fixtures: AccessFixture[] = [+ {+ id: "owned",+ owner: true,+ password: "video-hash",+ spacePasswords: ["space-hash"],+ },+ { id: "org-member", orgMember: true },+ { id: "space-member", spaceMember: true },+ { id: "public", public: true },+ { id: "private" },+ { id: "protected", orgMember: true, password: "video-hash" },+ {+ id: "inherited",+ spaceMember: true,+ spacePasswords: ["space-hash", "other-hash"],+ },+ { id: "invited", grant: true },+ { id: "restricted", public: true, domain: "example.com" },+ { id: "denied-domain", public: true, domain: "other.com" },+ { id: "domain-invited", public: true, domain: "other.com", grant: true },+];++function runPolicies(+ bulk: boolean,+ authenticated = true,+ passwords: string[] = [],+) {+ const loaded = (mocks.fixtures ?? []).map(+ (fixture) =>+ [+ {+ id: Video.VideoId.make(fixture.id),+ orgId: Organisation.OrganisationId.make(`org-${fixture.id}`),+ ownerId: User.UserId.make(fixture.owner ? "viewer" : "owner"),+ public: fixture.public ?? false,+ },+ Option.fromNullable(fixture.password),+ ] as const,+ );+ const program = Effect.flatMap(VideosPolicy, (policy) =>+ bulk+ ? policy.canViewLoadedBulk(loaded)+ : Effect.forEach(loaded, ([video, password]) =>+ policy.canViewLoaded(video, password).pipe(Effect.exit),+ ),+ );+ const withUser = authenticated+ ? program.pipe(Effect.provideService(CurrentUser, user))+ : program;+ return withUser.pipe(+ Effect.provide(VideosPolicy.Default),+ Effect.provideService(Video.VideoPasswordAttachment, { passwords }),+ Logger.withMinimumLogLevel(LogLevel.None),+ Effect.map((results) =>+ results.map((result) =>+ Exit.isSuccess(result)+ ? "allowed"+ : Option.getOrThrow(Cause.failureOption(result.cause))._tag,+ ),+ ),+ Effect.runPromise,+ );+}++describe("batched Caps access parity", () => {+ it.each([+ { authenticated: true, passwords: [] },+ { authenticated: false, passwords: [] },+ { authenticated: true, passwords: ["video-hash", "other-hash"] },+ { authenticated: false, passwords: ["video-hash", "other-hash"] },+ { authenticated: true, passwords: ["wrong-hash"] },+ ])(+ "matches individual policies for $authenticated and $passwords",+ async ({ authenticated, passwords }) => {+ mocks.fixtures = fixtures;+ const single = await runPolicies(false, authenticated, passwords);+ expect(await runPolicies(true, authenticated, passwords)).toEqual(single);+ },+ );++ it.each(["space_videos", "organization_members", "space_members"])(+ "preserves per-video failures from %s",+ async (table) => {+ mocks.fixtures = fixtures;+ mocks.failTable = table;+ const single = await runPolicies(false);+ expect(single[0]).toBe("allowed");+ expect(single.slice(1)).toContain("DatabaseError");+ expect(await runPolicies(true)).toEqual(single);+ },+ );++ it("rechecks access after membership and passwords change", async () => {+ mocks.fixtures = [{ id: "shared", spaceMember: true }];+ expect(await runPolicies(true)).toEqual(["allowed"]);+ mocks.fixtures = [{ id: "shared" }];+ expect(await runPolicies(true)).toEqual(["PolicyDenied"]);+ mocks.fixtures = [+ { id: "shared", spaceMember: true, spacePasswords: ["new-hash"] },+ ];+ expect(await runPolicies(true)).toEqual(["VerifyVideoPasswordError"]);+ });++ it("does not query an empty policy batch", async () => {+ mocks.fixtures = [];+ expect(await runPolicies(true)).toEqual([]);+ expect(mocks.query).not.toHaveBeenCalled();+ });+});diff --git a/apps/web/__tests__/unit/video-analytics-bulk.test.ts b/apps/web/__tests__/unit/video-analytics-bulk.test.tsindex 4d68e8d13..c03f42e37 100644--- a/apps/web/__tests__/unit/video-analytics-bulk.test.ts+++ b/apps/web/__tests__/unit/video-analytics-bulk.test.ts@@ -1,4 +1,10 @@-import { CurrentUser, Organisation, User, Video } from "@cap/web-domain";+import {+ CurrentUser,+ Organisation,+ Policy,+ User,+ Video,+} from "@cap/web-domain";import { Effect, Exit, Option } from "effect";import { beforeEach, describe, expect, it, vi } from "vitest";@@ -55,18 +61,24 @@ const { VideosRepo } = await import("@cap/web-backend/src/Videos/VideosRepo");const { Database } = await import("@cap/web-backend/src/Database");const { Storage } = await import("@cap/web-backend/src/Storage/index");const { Tinybird } = await import("@cap/web-backend/src/Tinybird/index");-const { Policy } = await import("@cap/web-domain");-const check = vi.fn((video: { id: string }) =>- video.id === "denied"- ? Effect.fail(new Policy.PolicyDeniedError())- : video.id === "protected"- ? Effect.fail(- new Video.VerifyVideoPasswordError({- id: Video.VideoId.make(video.id),- cause: "not-provided",- }),- )- : Effect.void,+const check = vi.fn(+ (+ video: { id: string },+ _password: Option.Option<string>,+ ): Effect.Effect<+ void,+ Policy.PolicyDeniedError | Video.VerifyVideoPasswordError+ > =>+ video.id === "denied"+ ? Effect.fail(new Policy.PolicyDeniedError())+ : video.id === "protected"+ ? Effect.fail(+ new Video.VerifyVideoPasswordError({+ id: Video.VideoId.make(video.id),+ cause: "not-provided",+ }),+ )+ : Effect.void,);const user: CurrentUser["Type"] = {id: User.UserId.make("owner"),@@ -85,6 +97,10 @@ function run(ids: string[]) {VideosPolicy,VideosPolicy.make({canViewLoaded: check,+ canViewLoadedBulk: (loaded) =>+ Effect.forEach(loaded, ([video, password]) =>+ check(video, password).pipe(Effect.exit),+ ),canView: vi.fn(),isOwner: vi.fn(),isOwnerLoaded: vi.fn(),diff --git a/packages/web-backend/src/Organisations/OrganisationsRepo.ts b/packages/web-backend/src/Organisations/OrganisationsRepo.tsindex 19db18843..a1368a01e 100644--- a/packages/web-backend/src/Organisations/OrganisationsRepo.ts+++ b/packages/web-backend/src/Organisations/OrganisationsRepo.ts@@ -12,6 +12,31 @@ export class OrganisationsRepo extends Effect.Service<OrganisationsRepo>()(const db = yield* Database;return {+ membershipsForVideos: (+ userId: User.UserId,+ videoIds: readonly Video.VideoId[],+ ) =>+ db.use((db) =>+ db+ .select({+ videoId: Db.sharedVideos.videoId,+ membershipId: Db.organizationMembers.id,+ })+ .from(Db.organizationMembers)+ .innerJoin(+ Db.sharedVideos,+ Dz.eq(+ Db.organizationMembers.organizationId,+ Db.sharedVideos.organizationId,+ ),+ )+ .where(+ Dz.and(+ Dz.eq(Db.organizationMembers.userId, userId),+ Dz.inArray(Db.sharedVideos.videoId, [...videoIds]),+ ),+ ),+ ),membershipForVideo: (userId: User.UserId, videoId: Video.VideoId) =>db.use((db) =>dbdiff --git a/packages/web-backend/src/Spaces/SpacesRepo.ts b/packages/web-backend/src/Spaces/SpacesRepo.tsindex 46035b912..6db938621 100644--- a/packages/web-backend/src/Spaces/SpacesRepo.ts+++ b/packages/web-backend/src/Spaces/SpacesRepo.ts@@ -10,6 +10,39 @@ export class SpacesRepo extends Effect.Service<SpacesRepo>()("SpacesRepo", {const db = yield* Database;return {+ membershipsForVideos: (+ userId: User.UserId,+ videoIds: readonly Video.VideoId[],+ ) =>+ db.use((db) =>+ db+ .select({+ videoId: Db.spaceVideos.videoId,+ membershipId: Db.spaceMembers.id,+ })+ .from(Db.spaceMembers)+ .innerJoin(+ Db.spaceVideos,+ Dz.eq(Db.spaceMembers.spaceId, Db.spaceVideos.spaceId),+ )+ .where(+ Dz.and(+ Dz.eq(Db.spaceMembers.userId, userId),+ Dz.inArray(Db.spaceVideos.videoId, [...videoIds]),+ ),+ ),+ ),+ passwordsForVideos: (videoIds: readonly Video.VideoId[]) =>+ db.use((db) =>+ db+ .select({+ videoId: Db.spaceVideos.videoId,+ password: Db.spaces.password,+ })+ .from(Db.spaceVideos)+ .innerJoin(Db.spaces, Dz.eq(Db.spaceVideos.spaceId, Db.spaces.id))+ .where(Dz.inArray(Db.spaceVideos.videoId, [...videoIds])),+ ),membershipForVideo: (userId: User.UserId, videoId: Video.VideoId) =>db.use((db) =>diff --git a/packages/web-backend/src/Videos/VideosPolicy.ts b/packages/web-backend/src/Videos/VideosPolicy.tsindex 1160dd0f9..c3270bdab 100644--- a/packages/web-backend/src/Videos/VideosPolicy.ts+++ b/packages/web-backend/src/Videos/VideosPolicy.ts@@ -1,6 +1,6 @@import { isEmailAllowedByRestriction } from "@cap/utils";import {- type CurrentUser,+ CurrentUser,type DatabaseError,type Organisation,Policy,@@ -192,6 +192,81 @@ export class VideosPolicy extends Effect.Service<VideosPolicy>()(password: Option.Option<string>,) => buildCanViewLoaded(deps, video, password);+ const canViewLoadedBulk = Effect.fn("VideosPolicy.canViewLoadedBulk")(+ function* (+ loaded: ReadonlyArray<+ readonly [ViewableVideo, Option.Option<string>]+ >,+ ) {+ const user = yield* Effect.serviceOption(CurrentUser);+ const videoIds = loaded+ .filter(+ ([video]) =>+ Option.isNone(user) || video.ownerId !== user.value.id,+ )+ .map(([video]) => video.id);+ if (videoIds.length === 0)+ return yield* Effect.forEach(loaded, ([video, password]) =>+ canViewLoaded(video, password).pipe(Effect.exit),+ );++ // Retain lookup failures until each policy runs, including the owner bypass.+ const [passwords, orgMemberships, spaceMemberships] =+ yield* Effect.all(+ [+ spacesRepo.passwordsForVideos(videoIds).pipe(+ Effect.map((rows) =>+ Array.groupBy(rows, (row) => row.videoId),+ ),+ Effect.exit,+ ),+ (Option.isSome(user)+ ? orgsRepo.membershipsForVideos(user.value.id, videoIds)+ : Effect.succeed([])+ ).pipe(+ Effect.map((rows) =>+ Array.groupBy(rows, (row) => row.videoId),+ ),+ Effect.exit,+ ),+ (Option.isSome(user)+ ? spacesRepo.membershipsForVideos(user.value.id, videoIds)+ : Effect.succeed([])+ ).pipe(+ Effect.map((rows) =>+ Array.groupBy(rows, (row) => row.videoId),+ ),+ Effect.exit,+ ),+ ],+ { concurrency: "unbounded" },+ );+ const bulkDeps: ViewDecisionDeps = {+ ...deps,+ orgsRepo: {+ ...orgsRepo,+ membershipForVideo: (_, id) =>+ Effect.map(orgMemberships, (rows) => rows[id] ?? []),+ },+ spacesRepo: {+ ...spacesRepo,+ passwordsForVideo: (id) =>+ Effect.map(passwords, (rows) => rows[id] ?? []),+ membershipForVideo: (_, id) =>+ Effect.map(spaceMemberships, (rows) =>+ Option.fromNullable(rows[id]?.[0]),+ ),+ },+ };+ return yield* Effect.forEach(+ loaded,+ ([video, password]) =>+ buildCanViewLoaded(bulkDeps, video, password).pipe(Effect.exit),+ { concurrency: 10 },+ );+ },+ );+const isOwner = (videoId: Video.VideoId) =>Policy.policy((user) =>repo.getById(videoId).pipe(@@ -234,6 +309,7 @@ export class VideosPolicy extends Effect.Service<VideosPolicy>()(return {canView,canViewLoaded,+ canViewLoadedBulk,isOwner,isOwnerLoaded,getViewableById,diff --git a/packages/web-backend/src/Videos/index.ts b/packages/web-backend/src/Videos/index.tsindex 2d27caa09..fc5c08a9d 100644--- a/packages/web-backend/src/Videos/index.ts+++ b/packages/web-backend/src/Videos/index.ts@@ -12,7 +12,7 @@ import {Video,} from "@cap/web-domain";import * as Dz from "drizzle-orm";-import { Effect, Exit, Option } from "effect";+import { Array as Arr, Effect, Exit, Option } from "effect";import type { Schema } from "effect/Schema";import { Database } from "../Database.ts";@@ -253,20 +253,23 @@ export class Videos extends Effect.Service<Videos>()("Videos", {.from(Db.videos).where(Dz.inArray(Db.videos.id, [...new Set(videoIds)])),);- const videosById = new Map(rows.map((video) => [video.id, video]));- const videoExits = yield* Effect.forEach(- videoIds,- (videoId) => {- const video = videosById.get(videoId);- if (!video)- return Effect.succeed(- Exit.succeed(Option.none<(typeof rows)[number]>()),- );- return policy- .canViewLoaded(video, Option.fromNullable(video.password))- .pipe(Effect.as(Option.some(video)), Effect.exit);- },- { concurrency: 10 },+ const policyExits = yield* policy.canViewLoadedBulk(+ rows.map(+ (video) => [video, Option.fromNullable(video.password)] as const,+ ),+ );+ const resultsById = new Map(+ Arr.zipWith(+ rows,+ policyExits,+ (video, exit) =>+ [video.id, Exit.map(exit, () => Option.some(video))] as const,+ ),+ );+ const videoExits = videoIds.map(+ (videoId) =>+ resultsById.get(videoId) ??+ Exit.succeed(Option.none<(typeof rows)[number]>()),);const analyticsVideos = videoExits.flatMap((exit) => {if (!Exit.isSuccess(exit) || Option.isNone(exit.value)) return [];