Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(web): batch access checks for Caps grid analytics

Aditya Garud87ffd7ed7734
diff --git a/apps/web/__tests__/unit/caps-analytics-round-trips.test.ts b/apps/web/__tests__/unit/caps-analytics-round-trips.test.ts
new file mode 100644
index 000000000..036a2b382
--- /dev/null
+++ b/apps/web/__tests__/unit/caps-analytics-round-trips.test.ts
@@ -0,0 +1,359 @@
+import { CurrentUser, Organisation, User, Video } from "@cap/web-domain";
+import { Cause, Effect, Exit, Logger, LogLevel, Option } from "effect";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+type AccessFixture = {
+ id: string;
+ public?: boolean;
+ owner?: boolean;
+ orgMember?: boolean;
+ spaceMember?: boolean;
+ password?: string;
+ spacePasswords?: string[];
+ domain?: string;
+ grant?: boolean;
+};
+const mocks = vi.hoisted(() => ({
+ query: vi.fn(),
+ latency: 0,
+ size: 50,
+ owner: false,
+ fixtures: null as AccessFixture[] | null,
+ failTable: "",
+}));
+vi.mock("@cap/database", async () => {
+ const { drizzle } = await import("drizzle-orm/mysql-proxy");
+ return { db: () => drizzle(mocks.query) };
+});
+vi.mock("@cap/utils", async () => {
+ const { isEmailAllowedByRestriction } = await import(
+ "../../../../packages/utils/src/helpers"
+ );
+ return { dub: {}, isEmailAllowedByRestriction };
+});
+vi.mock("@cap/web-backend/src/Storage/index", async () => {
+ const { Effect } = await import("effect");
+ class Storage extends Effect.Service<Storage>()("Storage", {
+ sync: () => ({}),
+ }) {}
+ return { Storage };
+});
+vi.mock("@cap/web-backend/src/Tinybird/index", async () => {
+ const { Effect } = await import("effect");
+ class Tinybird extends Effect.Service<Tinybird>()("Tinybird", {
+ succeed: {
+ querySql: () =>
+ Effect.succeed({
+ data: Array.from({ length: mocks.size }, (_, index) => ({
+ pathname: `/s/video-${index}`,
+ views: index + 1,
+ })),
+ }),
+ },
+ }) {}
+ return { Tinybird };
+});
+
+const { Videos } = await import("@cap/web-backend/src/Videos/index");
+const { VideosPolicy } = await import(
+ "@cap/web-backend/src/Videos/VideosPolicy"
+);
+const { VideosRepo } = await import("@cap/web-backend/src/Videos/VideosRepo");
+const { Database } = await import("@cap/web-backend/src/Database");
+const { Storage } = await import("@cap/web-backend/src/Storage/index");
+const { Tinybird } = await import("@cap/web-backend/src/Tinybird/index");
+
+const user: CurrentUser["Type"] = {
+ id: User.UserId.make("viewer"),
+ email: "[email protected]",
+ activeOrganizationId: Organisation.OrganisationId.make("org"),
+ iconUrlOrKey: Option.none(),
+};
+function ids() {
+ return Array.from({ length: mocks.size }, (_, index) =>
+ Video.VideoId.make(`video-${index}`),
+ );
+}
+function run() {
+ return Effect.flatMap(Videos, (videos) =>
+ videos.getAnalyticsBulk(ids()),
+ ).pipe(
+ Effect.provide(Videos.DefaultWithoutDependencies),
+ Effect.provide([
+ VideosPolicy.Default,
+ VideosRepo.Default,
+ Storage.Default,
+ Tinybird.Default,
+ Database.Default,
+ ]),
+ Effect.provideService(CurrentUser, user),
+ Logger.withMinimumLogLevel(LogLevel.None),
+ Effect.runPromise,
+ );
+}
+function reply(sql: string, params: unknown[] = []) {
+ if (mocks.failTable && sql.includes(`from \`${mocks.failTable}\``))
+ throw new Error("Database unavailable");
+ if (mocks.fixtures) {
+ const selected = mocks.fixtures.filter((fixture) =>
+ params.includes(fixture.id),
+ );
+ if (sql.includes("from `organization_members`")) {
+ expect(params[0]).toBe("viewer");
+ return selected
+ .filter((fixture) => fixture.orgMember)
+ .map((fixture) =>
+ sql.includes("`shared_videos`.`videoId`,")
+ ? [fixture.id, "org-membership"]
+ : ["org-membership"],
+ );
+ }
+ if (sql.includes("from `space_members`")) {
+ expect(params[0]).toBe("viewer");
+ return selected
+ .filter((fixture) => fixture.spaceMember)
+ .map((fixture) =>
+ sql.includes("`space_videos`.`videoId`,")
+ ? [fixture.id, "space-membership"]
+ : ["space-membership"],
+ );
+ }
+ if (sql.includes("from `space_videos`"))
+ return selected.flatMap((fixture) =>
+ (fixture.spacePasswords ?? []).map((password) =>
+ sql.startsWith("select `space_videos`.`videoId`")
+ ? [fixture.id, password]
+ : ["space", "Space", password],
+ ),
+ );
+ if (sql.includes("from `organizations`"))
+ return mocks.fixtures
+ .filter((fixture) => params.includes(`org-${fixture.id}`))
+ .map((fixture) => [fixture.domain ?? null]);
+ if (sql.includes("from `video_viewer_grants`"))
+ return selected.filter((fixture) => fixture.grant).map(() => ["grant"]);
+ throw new Error(`Unexpected policy query: ${sql}`);
+ }
+ if (sql.includes("from `videos`"))
+ return ids().map((id) => [
+ id,
+ "org",
+ mocks.owner ? "viewer" : "owner",
+ 0,
+ null,
+ ]);
+ if (sql.includes("from `organization_members`"))
+ return sql.includes("`shared_videos`.`videoId`,")
+ ? ids().map((id) => [id, "membership"])
+ : [["membership"]];
+ if (
+ sql.includes("from `space_members`") ||
+ sql.includes("from `space_videos`")
+ )
+ return [];
+ throw new Error(`Unexpected query: ${sql}`);
+}
+async function countRoundTrips() {
+ const pending: Array<() => void> = [];
+ mocks.query.mockImplementation(
+ (sql: string, params: unknown[]) =>
+ new Promise((resolve) =>
+ pending.push(() => resolve({ rows: reply(sql, params) })),
+ ),
+ );
+ let settled = false;
+ const result = run().finally(() => {
+ settled = true;
+ });
+ let rounds = 0;
+ let idleTicks = 0;
+ while (!settled) {
+ const issued = mocks.query.mock.calls.length;
+ await new Promise((resolve) => setTimeout(resolve, 0));
+ if (mocks.query.mock.calls.length !== issued || pending.length === 0) {
+ idleTicks = 0;
+ continue;
+ }
+ if (++idleTicks < 3) continue;
+ idleTicks = 0;
+ rounds++;
+ for (const release of pending.splice(0)) release();
+ }
+ await result;
+ return rounds;
+}
+function timedQueries() {
+ mocks.query.mockImplementation(async (sql: string, params: unknown[]) => {
+ if (mocks.latency)
+ await new Promise((resolve) => setTimeout(resolve, mocks.latency));
+ return { rows: reply(sql, params) };
+ });
+}
+beforeEach(() => {
+ mocks.size = 50;
+ mocks.fixtures = null;
+ mocks.failTable = "";
+ mocks.owner = false;
+ mocks.latency = 0;
+ timedQueries();
+});
+describe("Caps grid analytics database load", () => {
+ it.each([50, 200])("returns every shared count for %i Caps", async (size) => {
+ mocks.size = size;
+ const results = await run();
+ expect(results.map((result) => Exit.getOrElse(result, () => null))).toEqual(
+ ids().map((_, index) => ({ count: index + 1 })),
+ );
+ expect(mocks.query).toHaveBeenCalledTimes(4);
+ });
+ it("keeps owner analytics to one query", async () => {
+ mocks.owner = true;
+ await run();
+ expect(mocks.query).toHaveBeenCalledTimes(1);
+ });
+ it.skipIf(!process.env.CAP_BENCH_CAPS_ANALYTICS)(
+ "measures 20 ms database latency for My Caps and Shared Caps",
+ async () => {
+ for (const owner of [true, false]) {
+ for (const size of [15, 50, 200]) {
+ mocks.owner = owner;
+ mocks.size = size;
+ mocks.latency = 20;
+ const rounds = await countRoundTrips();
+ timedQueries();
+ const times: number[] = [];
+ let queries = 0;
+ for (let iteration = 0; iteration < 7; iteration++) {
+ mocks.query.mockClear();
+ const start = performance.now();
+ const results = await run();
+ times.push(performance.now() - start);
+ queries = mocks.query.mock.calls.length;
+ expect(results.every(Exit.isSuccess)).toBe(true);
+ }
+ times.sort((a, b) => a - b);
+ console.log(
+ JSON.stringify({
+ owner,
+ size,
+ queries,
+ rounds,
+ medianMs: times[3],
+ }),
+ );
+ }
+ }
+ },
+ );
+});
+
+const fixtures: AccessFixture[] = [
+ {
+ id: "owned",
+ owner: true,
+ password: "video-hash",
+ spacePasswords: ["space-hash"],
+ },
+ { id: "org-member", orgMember: true },
+ { id: "space-member", spaceMember: true },
+ { id: "public", public: true },
+ { id: "private" },
+ { id: "protected", orgMember: true, password: "video-hash" },
+ {
+ id: "inherited",
+ spaceMember: true,
+ spacePasswords: ["space-hash", "other-hash"],
+ },
+ { id: "invited", grant: true },
+ { id: "restricted", public: true, domain: "example.com" },
+ { id: "denied-domain", public: true, domain: "other.com" },
+ { id: "domain-invited", public: true, domain: "other.com", grant: true },
+];
+
+function runPolicies(
+ bulk: boolean,
+ authenticated = true,
+ passwords: string[] = [],
+) {
+ const loaded = (mocks.fixtures ?? []).map(
+ (fixture) =>
+ [
+ {
+ id: Video.VideoId.make(fixture.id),
+ orgId: Organisation.OrganisationId.make(`org-${fixture.id}`),
+ ownerId: User.UserId.make(fixture.owner ? "viewer" : "owner"),
+ public: fixture.public ?? false,
+ },
+ Option.fromNullable(fixture.password),
+ ] as const,
+ );
+ const program = Effect.flatMap(VideosPolicy, (policy) =>
+ bulk
+ ? policy.canViewLoadedBulk(loaded)
+ : Effect.forEach(loaded, ([video, password]) =>
+ policy.canViewLoaded(video, password).pipe(Effect.exit),
+ ),
+ );
+ const withUser = authenticated
+ ? program.pipe(Effect.provideService(CurrentUser, user))
+ : program;
+ return withUser.pipe(
+ Effect.provide(VideosPolicy.Default),
+ Effect.provideService(Video.VideoPasswordAttachment, { passwords }),
+ Logger.withMinimumLogLevel(LogLevel.None),
+ Effect.map((results) =>
+ results.map((result) =>
+ Exit.isSuccess(result)
+ ? "allowed"
+ : Option.getOrThrow(Cause.failureOption(result.cause))._tag,
+ ),
+ ),
+ Effect.runPromise,
+ );
+}
+
+describe("batched Caps access parity", () => {
+ it.each([
+ { authenticated: true, passwords: [] },
+ { authenticated: false, passwords: [] },
+ { authenticated: true, passwords: ["video-hash", "other-hash"] },
+ { authenticated: false, passwords: ["video-hash", "other-hash"] },
+ { authenticated: true, passwords: ["wrong-hash"] },
+ ])(
+ "matches individual policies for $authenticated and $passwords",
+ async ({ authenticated, passwords }) => {
+ mocks.fixtures = fixtures;
+ const single = await runPolicies(false, authenticated, passwords);
+ expect(await runPolicies(true, authenticated, passwords)).toEqual(single);
+ },
+ );
+
+ it.each(["space_videos", "organization_members", "space_members"])(
+ "preserves per-video failures from %s",
+ async (table) => {
+ mocks.fixtures = fixtures;
+ mocks.failTable = table;
+ const single = await runPolicies(false);
+ expect(single[0]).toBe("allowed");
+ expect(single.slice(1)).toContain("DatabaseError");
+ expect(await runPolicies(true)).toEqual(single);
+ },
+ );
+
+ it("rechecks access after membership and passwords change", async () => {
+ mocks.fixtures = [{ id: "shared", spaceMember: true }];
+ expect(await runPolicies(true)).toEqual(["allowed"]);
+ mocks.fixtures = [{ id: "shared" }];
+ expect(await runPolicies(true)).toEqual(["PolicyDenied"]);
+ mocks.fixtures = [
+ { id: "shared", spaceMember: true, spacePasswords: ["new-hash"] },
+ ];
+ expect(await runPolicies(true)).toEqual(["VerifyVideoPasswordError"]);
+ });
+
+ it("does not query an empty policy batch", async () => {
+ mocks.fixtures = [];
+ expect(await runPolicies(true)).toEqual([]);
+ expect(mocks.query).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/web/__tests__/unit/video-analytics-bulk.test.ts b/apps/web/__tests__/unit/video-analytics-bulk.test.ts
index 4d68e8d13..c03f42e37 100644
--- a/apps/web/__tests__/unit/video-analytics-bulk.test.ts
+++ b/apps/web/__tests__/unit/video-analytics-bulk.test.ts
@@ -1,4 +1,10 @@
-import { CurrentUser, Organisation, User, Video } from "@cap/web-domain";
+import {
+ CurrentUser,
+ Organisation,
+ Policy,
+ User,
+ Video,
+} from "@cap/web-domain";
import { Effect, Exit, Option } from "effect";
import { beforeEach, describe, expect, it, vi } from "vitest";
@@ -55,18 +61,24 @@ const { VideosRepo } = await import("@cap/web-backend/src/Videos/VideosRepo");
const { Database } = await import("@cap/web-backend/src/Database");
const { Storage } = await import("@cap/web-backend/src/Storage/index");
const { Tinybird } = await import("@cap/web-backend/src/Tinybird/index");
-const { Policy } = await import("@cap/web-domain");
-const check = vi.fn((video: { id: string }) =>
- video.id === "denied"
- ? Effect.fail(new Policy.PolicyDeniedError())
- : video.id === "protected"
- ? Effect.fail(
- new Video.VerifyVideoPasswordError({
- id: Video.VideoId.make(video.id),
- cause: "not-provided",
- }),
- )
- : Effect.void,
+const check = vi.fn(
+ (
+ video: { id: string },
+ _password: Option.Option<string>,
+ ): Effect.Effect<
+ void,
+ Policy.PolicyDeniedError | Video.VerifyVideoPasswordError
+ > =>
+ video.id === "denied"
+ ? Effect.fail(new Policy.PolicyDeniedError())
+ : video.id === "protected"
+ ? Effect.fail(
+ new Video.VerifyVideoPasswordError({
+ id: Video.VideoId.make(video.id),
+ cause: "not-provided",
+ }),
+ )
+ : Effect.void,
);
const user: CurrentUser["Type"] = {
id: User.UserId.make("owner"),
@@ -85,6 +97,10 @@ function run(ids: string[]) {
VideosPolicy,
VideosPolicy.make({
canViewLoaded: check,
+ canViewLoadedBulk: (loaded) =>
+ Effect.forEach(loaded, ([video, password]) =>
+ check(video, password).pipe(Effect.exit),
+ ),
canView: vi.fn(),
isOwner: vi.fn(),
isOwnerLoaded: vi.fn(),
diff --git a/packages/web-backend/src/Organisations/OrganisationsRepo.ts b/packages/web-backend/src/Organisations/OrganisationsRepo.ts
index 19db18843..a1368a01e 100644
--- a/packages/web-backend/src/Organisations/OrganisationsRepo.ts
+++ b/packages/web-backend/src/Organisations/OrganisationsRepo.ts
@@ -12,6 +12,31 @@ export class OrganisationsRepo extends Effect.Service<OrganisationsRepo>()(
const db = yield* Database;
return {
+ membershipsForVideos: (
+ userId: User.UserId,
+ videoIds: readonly Video.VideoId[],
+ ) =>
+ db.use((db) =>
+ db
+ .select({
+ videoId: Db.sharedVideos.videoId,
+ membershipId: Db.organizationMembers.id,
+ })
+ .from(Db.organizationMembers)
+ .innerJoin(
+ Db.sharedVideos,
+ Dz.eq(
+ Db.organizationMembers.organizationId,
+ Db.sharedVideos.organizationId,
+ ),
+ )
+ .where(
+ Dz.and(
+ Dz.eq(Db.organizationMembers.userId, userId),
+ Dz.inArray(Db.sharedVideos.videoId, [...videoIds]),
+ ),
+ ),
+ ),
membershipForVideo: (userId: User.UserId, videoId: Video.VideoId) =>
db.use((db) =>
db
diff --git a/packages/web-backend/src/Spaces/SpacesRepo.ts b/packages/web-backend/src/Spaces/SpacesRepo.ts
index 46035b912..6db938621 100644
--- a/packages/web-backend/src/Spaces/SpacesRepo.ts
+++ b/packages/web-backend/src/Spaces/SpacesRepo.ts
@@ -10,6 +10,39 @@ export class SpacesRepo extends Effect.Service<SpacesRepo>()("SpacesRepo", {
const db = yield* Database;
return {
+ membershipsForVideos: (
+ userId: User.UserId,
+ videoIds: readonly Video.VideoId[],
+ ) =>
+ db.use((db) =>
+ db
+ .select({
+ videoId: Db.spaceVideos.videoId,
+ membershipId: Db.spaceMembers.id,
+ })
+ .from(Db.spaceMembers)
+ .innerJoin(
+ Db.spaceVideos,
+ Dz.eq(Db.spaceMembers.spaceId, Db.spaceVideos.spaceId),
+ )
+ .where(
+ Dz.and(
+ Dz.eq(Db.spaceMembers.userId, userId),
+ Dz.inArray(Db.spaceVideos.videoId, [...videoIds]),
+ ),
+ ),
+ ),
+ passwordsForVideos: (videoIds: readonly Video.VideoId[]) =>
+ db.use((db) =>
+ db
+ .select({
+ videoId: Db.spaceVideos.videoId,
+ password: Db.spaces.password,
+ })
+ .from(Db.spaceVideos)
+ .innerJoin(Db.spaces, Dz.eq(Db.spaceVideos.spaceId, Db.spaces.id))
+ .where(Dz.inArray(Db.spaceVideos.videoId, [...videoIds])),
+ ),
membershipForVideo: (userId: User.UserId, videoId: Video.VideoId) =>
db
.use((db) =>
diff --git a/packages/web-backend/src/Videos/VideosPolicy.ts b/packages/web-backend/src/Videos/VideosPolicy.ts
index 1160dd0f9..c3270bdab 100644
--- a/packages/web-backend/src/Videos/VideosPolicy.ts
+++ b/packages/web-backend/src/Videos/VideosPolicy.ts
@@ -1,6 +1,6 @@
import { isEmailAllowedByRestriction } from "@cap/utils";
import {
- type CurrentUser,
+ CurrentUser,
type DatabaseError,
type Organisation,
Policy,
@@ -192,6 +192,81 @@ export class VideosPolicy extends Effect.Service<VideosPolicy>()(
password: Option.Option<string>,
) => buildCanViewLoaded(deps, video, password);
+ const canViewLoadedBulk = Effect.fn("VideosPolicy.canViewLoadedBulk")(
+ function* (
+ loaded: ReadonlyArray<
+ readonly [ViewableVideo, Option.Option<string>]
+ >,
+ ) {
+ const user = yield* Effect.serviceOption(CurrentUser);
+ const videoIds = loaded
+ .filter(
+ ([video]) =>
+ Option.isNone(user) || video.ownerId !== user.value.id,
+ )
+ .map(([video]) => video.id);
+ if (videoIds.length === 0)
+ return yield* Effect.forEach(loaded, ([video, password]) =>
+ canViewLoaded(video, password).pipe(Effect.exit),
+ );
+
+ // Retain lookup failures until each policy runs, including the owner bypass.
+ const [passwords, orgMemberships, spaceMemberships] =
+ yield* Effect.all(
+ [
+ spacesRepo.passwordsForVideos(videoIds).pipe(
+ Effect.map((rows) =>
+ Array.groupBy(rows, (row) => row.videoId),
+ ),
+ Effect.exit,
+ ),
+ (Option.isSome(user)
+ ? orgsRepo.membershipsForVideos(user.value.id, videoIds)
+ : Effect.succeed([])
+ ).pipe(
+ Effect.map((rows) =>
+ Array.groupBy(rows, (row) => row.videoId),
+ ),
+ Effect.exit,
+ ),
+ (Option.isSome(user)
+ ? spacesRepo.membershipsForVideos(user.value.id, videoIds)
+ : Effect.succeed([])
+ ).pipe(
+ Effect.map((rows) =>
+ Array.groupBy(rows, (row) => row.videoId),
+ ),
+ Effect.exit,
+ ),
+ ],
+ { concurrency: "unbounded" },
+ );
+ const bulkDeps: ViewDecisionDeps = {
+ ...deps,
+ orgsRepo: {
+ ...orgsRepo,
+ membershipForVideo: (_, id) =>
+ Effect.map(orgMemberships, (rows) => rows[id] ?? []),
+ },
+ spacesRepo: {
+ ...spacesRepo,
+ passwordsForVideo: (id) =>
+ Effect.map(passwords, (rows) => rows[id] ?? []),
+ membershipForVideo: (_, id) =>
+ Effect.map(spaceMemberships, (rows) =>
+ Option.fromNullable(rows[id]?.[0]),
+ ),
+ },
+ };
+ return yield* Effect.forEach(
+ loaded,
+ ([video, password]) =>
+ buildCanViewLoaded(bulkDeps, video, password).pipe(Effect.exit),
+ { concurrency: 10 },
+ );
+ },
+ );
+
const isOwner = (videoId: Video.VideoId) =>
Policy.policy((user) =>
repo.getById(videoId).pipe(
@@ -234,6 +309,7 @@ export class VideosPolicy extends Effect.Service<VideosPolicy>()(
return {
canView,
canViewLoaded,
+ canViewLoadedBulk,
isOwner,
isOwnerLoaded,
getViewableById,
diff --git a/packages/web-backend/src/Videos/index.ts b/packages/web-backend/src/Videos/index.ts
index 2d27caa09..fc5c08a9d 100644
--- a/packages/web-backend/src/Videos/index.ts
+++ b/packages/web-backend/src/Videos/index.ts
@@ -12,7 +12,7 @@ import {
Video,
} from "@cap/web-domain";
import * as Dz from "drizzle-orm";
-import { Effect, Exit, Option } from "effect";
+import { Array as Arr, Effect, Exit, Option } from "effect";
import type { Schema } from "effect/Schema";
import { Database } from "../Database.ts";
@@ -253,20 +253,23 @@ export class Videos extends Effect.Service<Videos>()("Videos", {
.from(Db.videos)
.where(Dz.inArray(Db.videos.id, [...new Set(videoIds)])),
);
- const videosById = new Map(rows.map((video) => [video.id, video]));
- const videoExits = yield* Effect.forEach(
- videoIds,
- (videoId) => {
- const video = videosById.get(videoId);
- if (!video)
- return Effect.succeed(
- Exit.succeed(Option.none<(typeof rows)[number]>()),
- );
- return policy
- .canViewLoaded(video, Option.fromNullable(video.password))
- .pipe(Effect.as(Option.some(video)), Effect.exit);
- },
- { concurrency: 10 },
+ const policyExits = yield* policy.canViewLoadedBulk(
+ rows.map(
+ (video) => [video, Option.fromNullable(video.password)] as const,
+ ),
+ );
+ const resultsById = new Map(
+ Arr.zipWith(
+ rows,
+ policyExits,
+ (video, exit) =>
+ [video.id, Exit.map(exit, () => Option.some(video))] as const,
+ ),
+ );
+ const videoExits = videoIds.map(
+ (videoId) =>
+ resultsById.get(videoId) ??
+ Exit.succeed(Option.none<(typeof rows)[number]>()),
);
const analyticsVideos = videoExits.flatMap((exit) => {
if (!Exit.isSuccess(exit) || Option.isNone(exit.value)) return [];