Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

perf(loops): batch unchanged Loops profile checks

Richie McIlroyc8b7dbfbab23
diff --git a/.github/workflows/database-profile-reliability.yml b/.github/workflows/database-profile-reliability.yml
index 83c853aae..c86795489 100644
--- a/.github/workflows/database-profile-reliability.yml
+++ b/.github/workflows/database-profile-reliability.yml
@@ -53,6 +53,7 @@ jobs:
run: >-
bun test
scripts/loops/activation.test.ts
+ scripts/loops/batch.test.ts
scripts/loops/sources.test.ts
scripts/loops/profile.test.ts
scripts/loops/lifecycle.test.ts
diff --git a/packages/database/loops/activation.ts b/packages/database/loops/activation.ts
index 84f1c3057..ebe3f04fb 100644
--- a/packages/database/loops/activation.ts
+++ b/packages/database/loops/activation.ts
@@ -1,11 +1,21 @@
+import { placeholders } from "./sql";
+
export const legacyActivationQuery = `
SELECT
EXISTS(SELECT 1 FROM videos WHERE ownerId = ? LIMIT 1) AS hasVideo,
EXISTS(SELECT 1 FROM videos WHERE ownerId = ? AND public = 1 LIMIT 1) AS hasSharedVideo
`;
-export const activationQuery = `
+export const batchLegacyActivationQuery = (users: number) => `
SELECT
+ u.id AS userId,
+ EXISTS(SELECT 1 FROM videos WHERE ownerId = u.id LIMIT 1) AS hasVideo,
+ EXISTS(SELECT 1 FROM videos WHERE ownerId = u.id AND public = 1 LIMIT 1) AS hasSharedVideo
+ FROM users u
+ WHERE u.id IN (${placeholders(users)})
+`;
+
+const activationSignals = `
MAX(CASE WHEN u.video_id IS NOT NULL OR (j.video_id IS NOT NULL AND j.state <> 'verified') THEN 1 ELSE 0 END) AS hasPendingUpload,
MAX(CASE WHEN u.video_id IS NULL
AND v.duration > 0
@@ -24,6 +34,15 @@ export const activationQuery = `
MAX(GREATEST(DATE_ADD(v.createdAt, INTERVAL 5 MINUTE), COALESCE(v.firstViewEmailSentAt, v.createdAt))) AS lastActivationNotificationAt
FROM videos v
LEFT JOIN video_uploads u ON u.video_id = v.id
- LEFT JOIN video_processing_jobs j ON j.video_id = v.id
+ LEFT JOIN video_processing_jobs j ON j.video_id = v.id`;
+
+export const activationQuery = `
+ SELECT ${activationSignals}
WHERE v.ownerId = ? AND v.isScreenshot = 0
`;
+
+export const batchActivationQuery = (users: number) => `
+ SELECT v.ownerId AS userId, ${activationSignals}
+ WHERE v.ownerId IN (${placeholders(users)}) AND v.isScreenshot = 0
+ GROUP BY v.ownerId
+`;
diff --git a/packages/database/loops/jobs.ts b/packages/database/loops/jobs.ts
new file mode 100644
index 000000000..915c77b44
--- /dev/null
+++ b/packages/database/loops/jobs.ts
@@ -0,0 +1,121 @@
+import type {
+ Connection,
+ ResultSetHeader,
+ RowDataPacket,
+} from "mysql2/promise";
+import { placeholders } from "./sql";
+
+export type Job = RowDataPacket & {
+ userId: string;
+ revision: number;
+ failures: number;
+ syncedEmail: string | null;
+ lastSyncedAt: string | null;
+ profileHash: string | null;
+ teammateJoinedAt: string | null;
+};
+
+export async function claimJobs(
+ database: Connection,
+ token: string,
+ allowedUserIds: string[] | null,
+ limit: number,
+) {
+ await database.beginTransaction();
+ try {
+ const [jobs] = await database.query<Job[]>(
+ `SELECT * FROM loops_sync_jobs WHERE nextAttemptAt<=UTC_TIMESTAMP() AND (leaseUntil IS NULL OR leaseUntil<UTC_TIMESTAMP()) ${allowedUserIds ? "AND userId IN (?)" : ""} ORDER BY nextAttemptAt LIMIT ? FOR UPDATE SKIP LOCKED`,
+ allowedUserIds ? [allowedUserIds, limit] : [limit],
+ );
+ if (jobs.length)
+ await database.execute(
+ `UPDATE loops_sync_jobs SET leaseToken=?,leaseUntil=DATE_ADD(UTC_TIMESTAMP(),INTERVAL 2 MINUTE) WHERE userId IN (${placeholders(jobs.length)})`,
+ [token, ...jobs.map((job) => job.userId)],
+ );
+ await database.commit();
+ return jobs;
+ } catch (error) {
+ await database.rollback();
+ throw error;
+ }
+}
+
+export async function renewLease(
+ database: Connection,
+ job: Pick<Job, "userId">,
+ token: string,
+) {
+ const [result] = await database.execute<ResultSetHeader>(
+ "UPDATE loops_sync_jobs SET leaseUntil=DATE_ADD(UTC_TIMESTAMP(),INTERVAL 2 MINUTE) WHERE userId=? AND leaseToken=?",
+ [job.userId, token],
+ );
+ return result.affectedRows > 0;
+}
+
+export async function releaseJobs(
+ database: Connection,
+ token: string,
+ userIds: string[],
+) {
+ await database.execute(
+ `UPDATE loops_sync_jobs SET leaseToken=NULL,leaseUntil=NULL WHERE leaseToken=? AND userId IN (${placeholders(userIds.length)})`,
+ [token, ...userIds],
+ );
+}
+
+export async function deferJob(
+ database: Connection,
+ job: Pick<Job, "userId" | "revision">,
+ token: string,
+ reason: string,
+ nextAttemptAt: Date,
+) {
+ await database.execute(
+ "UPDATE loops_sync_jobs SET failures=CASE WHEN revision=? THEN failures+1 ELSE 0 END,lastError=CASE WHEN revision=? THEN ? ELSE NULL END,nextAttemptAt=CASE WHEN revision=? THEN ? ELSE UTC_TIMESTAMP() END,leaseToken=NULL,leaseUntil=NULL WHERE userId=? AND leaseToken=?",
+ [
+ job.revision,
+ job.revision,
+ reason,
+ job.revision,
+ nextAttemptAt,
+ job.userId,
+ token,
+ ],
+ );
+}
+
+export async function completeUnchangedJob(
+ database: Connection,
+ job: Pick<Job, "userId" | "revision">,
+ token: string,
+ nextAttemptAt: Date,
+) {
+ await database.execute(
+ "UPDATE loops_sync_jobs SET failures=0,lastError=NULL,nextAttemptAt=CASE WHEN revision=? THEN ? ELSE UTC_TIMESTAMP() END,leaseToken=NULL,leaseUntil=NULL WHERE userId=? AND leaseToken=?",
+ [job.revision, nextAttemptAt, job.userId, token],
+ );
+}
+
+export async function completeUnchangedJobs(
+ database: Connection,
+ token: string,
+ jobs: { job: Pick<Job, "userId" | "revision">; nextAttemptAt: Date }[],
+) {
+ const groups = new Map<number, typeof jobs>();
+ for (const entry of jobs) {
+ const key = entry.nextAttemptAt.getTime();
+ const group = groups.get(key);
+ if (group) group.push(entry);
+ else groups.set(key, [entry]);
+ }
+ for (const group of groups.values())
+ await database.execute(
+ `UPDATE loops_sync_jobs SET failures=0,lastError=NULL,nextAttemptAt=CASE WHEN (userId,revision) IN (${group.map(() => "(?,?)").join(",")}) THEN ? ELSE UTC_TIMESTAMP() END,leaseToken=NULL,leaseUntil=NULL WHERE leaseToken=? AND userId IN (${placeholders(group.length)})`,
+ [
+ ...group.flatMap(({ job }) => [job.userId, job.revision]),
+ group[0].nextAttemptAt,
+ token,
+ ...group.map(({ job }) => job.userId),
+ ],
+ );
+}
diff --git a/packages/database/loops/sources.ts b/packages/database/loops/sources.ts
index bf02f12ec..f26c0db92 100644
--- a/packages/database/loops/sources.ts
+++ b/packages/database/loops/sources.ts
@@ -1,5 +1,10 @@
import type { Connection, RowDataPacket } from "mysql2/promise";
-import { activationQuery, legacyActivationQuery } from "./activation";
+import {
+ activationQuery,
+ batchActivationQuery,
+ batchLegacyActivationQuery,
+ legacyActivationQuery,
+} from "./activation";
import { activationSignalsAfter, inFreeExperiment } from "./experiment";
import {
type CapUser,
@@ -8,6 +13,7 @@ import {
type Membership,
type ProfileInput,
} from "./profile";
+import { placeholders } from "./sql";
type RuntimeUser = CapUser & {
emailVerified: string | null;
@@ -21,7 +27,7 @@ type UserSignals = {
hasPendingInvite: number;
};
-export const profileUserQuery = `
+const profileUserColumns = `
SELECT u.id,u.email,u.name,u.lastName,u.emailVerified,u.stripeCustomerId,
u.stripeSubscriptionStatus,u.thirdPartyStripeSubscriptionId,u.inviteQuota,u.created_at,
u.defaultOrgId,u.marketingOrigin,
@@ -29,10 +35,28 @@ export const profileUserQuery = `
EXISTS(SELECT 1 FROM accounts a WHERE a.userId = u.id AND BINARY a.provider = 'workos' LIMIT 1) AS hasWorkosAccount,
EXISTS(SELECT 1 FROM organization_invites i WHERE i.invitedEmail = u.email AND i.status IN ('pending','accepted') LIMIT 1) AS hasInvite,
EXISTS(SELECT 1 FROM organization_invites i WHERE i.invitedEmail = u.email AND BINARY i.status = 'pending' LIMIT 1) AS hasPendingInvite
- FROM users u
+ FROM users u`;
+
+export const profileUserQuery = `${profileUserColumns}
WHERE u.id = ?
`;
+export const batchProfileUserQuery = (users: number) => `${profileUserColumns}
+ WHERE u.id IN (${placeholders(users)})
+`;
+
+const membershipColumns =
+ "SELECT m.userId,m.organizationId,m.hasProSeat,o.ownerId,o.tombstoneAt,o.workosConnectionId,o.createdAt FROM organization_members m JOIN organizations o ON o.id=m.organizationId";
+
+export const batchMembershipQuery = (users: number) =>
+ `${membershipColumns} WHERE m.userId IN (${placeholders(users)})`;
+
+const licenseColumns = (kind: License["kind"], table: string, emails: string) =>
+ `SELECT FIELD(u.email,${emails}) AS requestIndex,u.email,l.subscriptionActive,l.nextRenewalDate,'${kind}' AS kind FROM ${table} l JOIN user u ON u.id=l.userId WHERE u.email IN (${emails})`;
+
+export const batchLicenseQuery = (emails: number) =>
+ `${licenseColumns("desktop", "commercialLicenses", placeholders(emails))} UNION ALL ${licenseColumns("selfhosted", "selfHostedLicenses", placeholders(emails))}`;
+
export type LoopsProfileSource = {
input: ProfileInput & { user: CapUser };
signedUp: boolean;
@@ -45,6 +69,49 @@ async function rows<T>(database: Connection, query: string, values: string[]) {
return result;
}
+type Activation = {
+ hasVideo: number | boolean | null;
+ hasSharedVideo: number | boolean | null;
+ hasPendingUpload?: number | null;
+ lastActivationNotificationAt?: string | null;
+};
+
+const noActivation: Activation = {
+ hasPendingUpload: null,
+ hasVideo: null,
+ hasSharedVideo: null,
+ lastActivationNotificationAt: null,
+};
+
+function profileSource(
+ record: RuntimeUser & UserSignals,
+ memberships: Membership[],
+ activation: Activation | undefined,
+ licenses: License[],
+ now: Date,
+): LoopsProfileSource {
+ const { hasAccount, hasWorkosAccount, hasInvite, hasPendingInvite, ...user } =
+ record;
+ return {
+ input: {
+ source: { email: user.email },
+ user,
+ memberships,
+ licenses,
+ invited: Boolean(hasInvite),
+ sso: Boolean(hasWorkosAccount),
+ hasVideo: Boolean(activation?.hasVideo),
+ hasSharedVideo: Boolean(activation?.hasSharedVideo),
+ lastActivationNotificationAt: activation?.lastActivationNotificationAt,
+ hasPendingUpload: Boolean(activation?.hasPendingUpload),
+ now,
+ },
+ signedUp: Boolean(user.emailVerified || hasAccount),
+ pendingInvite: Boolean(hasPendingInvite),
+ stripeCustomerId: user.stripeCustomerId,
+ };
+}
+
export async function readLoopsProfile(
cap: Connection,
licenses: Connection,
@@ -56,52 +123,123 @@ export async function readLoopsProfile(
[userId],
);
if (!record) return null;
- const { hasAccount, hasWorkosAccount, hasInvite, hasPendingInvite, ...user } =
- record;
const memberships = await rows<Membership>(
cap,
- "SELECT m.userId,m.organizationId,m.hasProSeat,o.ownerId,o.tombstoneAt,o.workosConnectionId,o.createdAt FROM organization_members m JOIN organizations o ON o.id=m.organizationId WHERE m.userId=?",
+ `${membershipColumns} WHERE m.userId=?`,
[userId],
);
- let activation: {
- hasVideo: number | boolean | null;
- hasSharedVideo: number | boolean | null;
- hasPendingUpload?: number | null;
- lastActivationNotificationAt?: string | null;
- };
- if (inFreeExperiment(isoDate(user.created_at), activationSignalsAfter)) {
- const [result] = await rows<typeof activation>(cap, activationQuery, [
- userId,
- ]);
- activation = result;
- } else {
- const [result] = await rows<typeof activation>(cap, legacyActivationQuery, [
- userId,
- userId,
- ]);
- activation = result;
- }
+ const [activation] = inFreeExperiment(
+ isoDate(record.created_at),
+ activationSignalsAfter,
+ )
+ ? await rows<Activation>(cap, activationQuery, [userId])
+ : await rows<Activation>(cap, legacyActivationQuery, [userId, userId]);
const entitlement = await rows<License>(
licenses,
"SELECT u.email,l.subscriptionActive,l.nextRenewalDate,'desktop' AS kind FROM commercialLicenses l JOIN user u ON u.id=l.userId WHERE u.email=? UNION ALL SELECT u.email,l.subscriptionActive,l.nextRenewalDate,'selfhosted' AS kind FROM selfHostedLicenses l JOIN user u ON u.id=l.userId WHERE u.email=?",
- [user.email, user.email],
+ [record.email, record.email],
);
- return {
- input: {
- source: { email: user.email },
- user,
- memberships,
- licenses: entitlement,
- invited: Boolean(hasInvite),
- sso: Boolean(hasWorkosAccount),
- hasVideo: Boolean(activation?.hasVideo),
- hasSharedVideo: Boolean(activation?.hasSharedVideo),
- lastActivationNotificationAt: activation?.lastActivationNotificationAt,
- hasPendingUpload: Boolean(activation?.hasPendingUpload),
- now: new Date(),
- },
- signedUp: Boolean(user.emailVerified || hasAccount),
- pendingInvite: Boolean(hasPendingInvite),
- stripeCustomerId: user.stripeCustomerId,
- };
+ return profileSource(
+ record,
+ memberships,
+ activation,
+ entitlement,
+ new Date(),
+ );
+}
+
+// Ids are matched case-insensitively, like the column collation the
+// single-user reads compare with.
+const idKey = (id: string) => id.toLowerCase();
+
+function groupBy<T>(items: T[], key: (item: T) => string) {
+ const groups = new Map<string, T[]>();
+ for (const item of items) {
+ const group = groups.get(key(item));
+ if (group) group.push(item);
+ else groups.set(key(item), [item]);
+ }
+ return groups;
+}
+
+export async function readLoopsProfiles(
+ cap: Connection,
+ licenses: Connection,
+ userIds: string[],
+): Promise<Map<string, LoopsProfileSource>> {
+ const profiles = new Map<string, LoopsProfileSource>();
+ if (!userIds.length) return profiles;
+ const records = await rows<RuntimeUser & UserSignals>(
+ cap,
+ batchProfileUserQuery(userIds.length),
+ userIds,
+ );
+ if (!records.length) return profiles;
+ const ids = records.map((record) => record.id);
+ const recent = records
+ .filter((record) =>
+ inFreeExperiment(isoDate(record.created_at), activationSignalsAfter),
+ )
+ .map((record) => record.id);
+ const legacy = records
+ .filter(
+ (record) =>
+ !inFreeExperiment(isoDate(record.created_at), activationSignalsAfter),
+ )
+ .map((record) => record.id);
+ const emails = records.map((record) => record.email);
+ const [memberships, recentActivation, legacyActivation, entitlements] =
+ await Promise.all([
+ rows<Membership>(cap, batchMembershipQuery(ids.length), ids),
+ recent.length
+ ? rows<Activation & { userId: string }>(
+ cap,
+ batchActivationQuery(recent.length),
+ recent,
+ )
+ : [],
+ legacy.length
+ ? rows<Activation & { userId: string }>(
+ cap,
+ batchLegacyActivationQuery(legacy.length),
+ legacy,
+ )
+ : [],
+ rows<License & { requestIndex: number }>(
+ licenses,
+ batchLicenseQuery(emails.length),
+ [...emails, ...emails, ...emails, ...emails],
+ ),
+ ]);
+ const membershipsByUser = groupBy(memberships, (row) => idKey(row.userId));
+ const activationByUser = new Map(
+ [...recentActivation, ...legacyActivation].map(
+ ({ userId, ...activation }) => [idKey(userId), activation],
+ ),
+ );
+ const licensesByEmail = groupBy(
+ entitlements,
+ (row) => emails[row.requestIndex - 1] ?? "",
+ );
+ const recordsById = new Map(
+ records.map((record) => [idKey(record.id), record]),
+ );
+ const now = new Date();
+ for (const userId of userIds) {
+ const record = recordsById.get(idKey(userId));
+ if (!record) continue;
+ profiles.set(
+ userId,
+ profileSource(
+ record,
+ membershipsByUser.get(idKey(record.id)) ?? [],
+ activationByUser.get(idKey(record.id)) ?? noActivation,
+ (licensesByEmail.get(record.email) ?? []).map(
+ ({ requestIndex: _, ...license }) => license,
+ ),
+ now,
+ ),
+ );
+ }
+ return profiles;
}
diff --git a/packages/database/loops/sql.ts b/packages/database/loops/sql.ts
new file mode 100644
index 000000000..a591f8f1d
--- /dev/null
+++ b/packages/database/loops/sql.ts
@@ -0,0 +1,3 @@
+export function placeholders(count: number) {
+ return Array.from({ length: count }, () => "?").join(",");
+}
diff --git a/packages/database/loops/worker.ts b/packages/database/loops/worker.ts
index 541a0e52a..39d37c0fd 100644
--- a/packages/database/loops/worker.ts
+++ b/packages/database/loops/worker.ts
@@ -2,7 +2,7 @@ import { createHash, randomUUID } from "node:crypto";
import { STRIPE_AVAILABLE } from "@cap/utils";
import { User } from "@cap/web-domain";
import { eq } from "drizzle-orm";
-import mysql, { type Connection, type RowDataPacket } from "mysql2/promise";
+import mysql, { type RowDataPacket } from "mysql2/promise";
import { provisionStripeCustomer } from "../auth/stripe-customer";
import { db } from "../index";
import { users } from "../schema";
@@ -12,6 +12,15 @@ import {
LoopsRequestError,
} from "./client";
import { activationSignalsAfter, inFreeExperiment } from "./experiment";
+import {
+ claimJobs,
+ completeUnchangedJob,
+ completeUnchangedJobs,
+ deferJob,
+ type Job,
+ releaseJobs,
+ renewLease,
+} from "./jobs";
import {
activationFollowUps,
enrollmentWindow,
@@ -22,18 +31,14 @@ import {
retryDelay,
} from "./lifecycle";
import { type CustomerCopy, classifyProfile, normalizeEmail } from "./profile";
-import { type LoopsProfileSource, readLoopsProfile } from "./sources";
+import {
+ type LoopsProfileSource,
+ readLoopsProfile,
+ readLoopsProfiles,
+} from "./sources";
import { profileFingerprint } from "./sync-policy";
-type Job = RowDataPacket & {
- userId: string;
- revision: number;
- failures: number;
- syncedEmail: string | null;
- lastSyncedAt: string | null;
- profileHash: string | null;
- teammateJoinedAt: string | null;
-};
+const claimBatchSize = 50;
export async function syncLoopsContact(
api: LoopsClient,
@@ -104,62 +109,79 @@ async function holdRemovedIdentity(
});
}
-export async function claimJob(
- database: Connection,
- token: string,
- allowedUserIds: string[] | null,
-) {
- await database.beginTransaction();
- try {
- const [jobs] = await database.query<Job[]>(
- `SELECT * FROM loops_sync_jobs WHERE nextAttemptAt<=UTC_TIMESTAMP() AND (leaseUntil IS NULL OR leaseUntil<UTC_TIMESTAMP()) ${allowedUserIds ? "AND userId IN (?)" : ""} ORDER BY nextAttemptAt LIMIT 1 FOR UPDATE SKIP LOCKED`,
- allowedUserIds ? [allowedUserIds] : [],
- );
- const job = jobs[0];
- if (job)
- await database.execute(
- "UPDATE loops_sync_jobs SET leaseToken=?,leaseUntil=DATE_ADD(UTC_TIMESTAMP(),INTERVAL 2 MINUTE) WHERE userId=?",
- [token, job.userId],
- );
- await database.commit();
- return job;
- } catch (error) {
- await database.rollback();
- throw error;
- }
+function identityChanged(
+ job: Job,
+ source: LoopsProfileSource | null,
+): job is Job & { syncedEmail: string } {
+ return Boolean(
+ job.syncedEmail &&
+ (!source ||
+ normalizeEmail(source.input.source.email) !== job.syncedEmail),
+ );
}
-export async function deferJob(
- database: Connection,
- job: Pick<Job, "userId" | "revision">,
- token: string,
- reason: string,
- nextAttemptAt: Date,
-) {
- await database.execute(
- "UPDATE loops_sync_jobs SET failures=CASE WHEN revision=? THEN failures+1 ELSE 0 END,lastError=CASE WHEN revision=? THEN ? ELSE NULL END,nextAttemptAt=CASE WHEN revision=? THEN ? ELSE UTC_TIMESTAMP() END,leaseToken=NULL,leaseUntil=NULL WHERE userId=? AND leaseToken=?",
- [
- job.revision,
- job.revision,
- reason,
- job.revision,
- nextAttemptAt,
- job.userId,
- token,
- ],
+function needsStripeCustomer(source: LoopsProfileSource) {
+ return (
+ source.signedUp &&
+ !source.input.sso &&
+ !source.stripeCustomerId &&
+ STRIPE_AVAILABLE()
);
}
-export async function completeUnchangedJob(
- database: Connection,
- job: Pick<Job, "userId" | "revision">,
- token: string,
- nextAttemptAt: Date,
+function syncFingerprint(
+ job: Job,
+ source: LoopsProfileSource,
+ config: LoopsRuntimeConfig,
+ customerCopy: CustomerCopy,
) {
- await database.execute(
- "UPDATE loops_sync_jobs SET failures=0,lastError=NULL,nextAttemptAt=CASE WHEN revision=? THEN ? ELSE UTC_TIMESTAMP() END,leaseToken=NULL,leaseUntil=NULL WHERE userId=? AND leaseToken=?",
- [job.revision, nextAttemptAt, job.userId, token],
+ const localProfile = classifyProfile(source.input, customerCopy);
+ const jobConfig = {
+ ...config,
+ teammateJoinedAt: job.teammateJoinedAt,
+ };
+ const hash = createHash("sha256")
+ .update(
+ JSON.stringify({
+ profile: profileFingerprint(localProfile),
+ signedUp: source.signedUp,
+ pendingInvite: source.pendingInvite,
+ enrollment: enrollmentWindow(localProfile.capSignupAt, jobConfig),
+ listId: config.listId,
+ teammateJoinedAt: job.teammateJoinedAt,
+ ...(inFreeExperiment(localProfile.capSignupAt, activationSignalsAfter)
+ ? {
+ freeExperiment: activationFollowUps(source.input),
+ freeExperimentEnrollmentEnabled:
+ config.freeExperimentEnrollmentEnabled &&
+ inFreeExperiment(
+ localProfile.capSignupAt,
+ config.freeExperimentAfter,
+ ),
+ }
+ : {}),
+ }),
+ )
+ .digest("hex");
+ return { localProfile, jobConfig, hash };
+}
+
+function unchangedUntil(
+ job: Job,
+ source: LoopsProfileSource | null,
+ config: LoopsRuntimeConfig,
+ customerCopy: CustomerCopy,
+ now: Date,
+) {
+ if (!source || identityChanged(job, source) || needsStripeCustomer(source))
+ return null;
+ const { localProfile, hash } = syncFingerprint(
+ job,
+ source,
+ config,
+ customerCopy,
);
+ return job.profileHash === hash ? nextProfileCheck(localProfile, now) : null;
}
export function loopsRuntimeConfig(env: NodeJS.ProcessEnv): LoopsRuntimeConfig {
@@ -235,28 +257,15 @@ export async function runLoopsSync(customerCopy: CustomerCopy) {
allowedUserIds = owned.map((user) => user.id);
if (!allowedUserIds.length) return { enabled: true, processed, failed };
}
- const deadline = Date.now() + 40_000;
- while (processed + failed < 500 && Date.now() < deadline) {
- const token = randomUUID();
- const job = await claimJob(cap, token, allowedUserIds);
- if (!job) break;
+ const syncJob = async (job: Job, token: string) => {
try {
let source = await readLoopsProfile(cap, license, job.userId);
- if (
- job.syncedEmail &&
- (!source ||
- normalizeEmail(source.input.source.email) !== job.syncedEmail)
- ) {
+ if (identityChanged(job, source)) {
await holdRemovedIdentity(api, job.syncedEmail, job.userId);
throw new Error("identity_change_held");
}
if (!source) throw new Error("user_removed");
- if (
- source.signedUp &&
- !source.input.sso &&
- !source.stripeCustomerId &&
- STRIPE_AVAILABLE()
- ) {
+ if (needsStripeCustomer(source)) {
const [user] = await db()
.select()
.from(users)
@@ -267,40 +276,12 @@ export async function runLoopsSync(customerCopy: CustomerCopy) {
source = await readLoopsProfile(cap, license, job.userId);
if (!source) throw new Error("user_removed");
}
- const localProfile = classifyProfile(source.input, customerCopy);
- const jobConfig = {
- ...config,
- teammateJoinedAt: job.teammateJoinedAt,
- };
- const hash = createHash("sha256")
- .update(
- JSON.stringify({
- profile: profileFingerprint(localProfile),
- signedUp: source.signedUp,
- pendingInvite: source.pendingInvite,
- enrollment: enrollmentWindow(
- localProfile.capSignupAt,
- jobConfig,
- ),
- listId: config.listId,
- teammateJoinedAt: job.teammateJoinedAt,
- ...(inFreeExperiment(
- localProfile.capSignupAt,
- activationSignalsAfter,
- )
- ? {
- freeExperiment: activationFollowUps(source.input),
- freeExperimentEnrollmentEnabled:
- config.freeExperimentEnrollmentEnabled &&
- inFreeExperiment(
- localProfile.capSignupAt,
- config.freeExperimentAfter,
- ),
- }
- : {}),
- }),
- )
- .digest("hex");
+ const { localProfile, jobConfig, hash } = syncFingerprint(
+ job,
+ source,
+ config,
+ customerCopy,
+ );
if (job.profileHash === hash) {
await completeUnchangedJob(
cap,
@@ -308,8 +289,7 @@ export async function runLoopsSync(customerCopy: CustomerCopy) {
token,
nextProfileCheck(localProfile, new Date()),
);
- processed++;
- continue;
+ return "unchanged";
}
const update = await syncLoopsContact(
api,
@@ -328,7 +308,7 @@ export async function runLoopsSync(customerCopy: CustomerCopy) {
"UPDATE loops_sync_jobs SET syncedEmail=?,profileHash=?,lastSyncedAt=UTC_TIMESTAMP(),failures=0,lastError=NULL,nextAttemptAt=CASE WHEN revision=? THEN ? ELSE UTC_TIMESTAMP() END,leaseToken=NULL,leaseUntil=NULL WHERE userId=? AND leaseToken=?",
[update.email, hash, job.revision, next, job.userId, token],
);
- processed++;
+ return "synced";
} catch (error) {
const reason =
error instanceof LoopsRequestError
@@ -344,14 +324,69 @@ export async function runLoopsSync(customerCopy: CustomerCopy) {
new Date(Date.now() + retryDelay(job.failures)),
);
console.error("Loops sync deferred", { userId: job.userId, reason });
- failed++;
- if (
- error instanceof LoopsRequestError &&
+ return error instanceof LoopsRequestError &&
[401, 403, 429].includes(error.status)
- )
- break;
+ ? "stop"
+ : "failed";
+ }
+ };
+ const deadline = Date.now() + 40_000;
+ let stopped = false;
+ while (!stopped && processed + failed < 500 && Date.now() < deadline) {
+ const token = randomUUID();
+ const jobs = await claimJobs(
+ cap,
+ token,
+ allowedUserIds,
+ Math.min(claimBatchSize, 500 - processed - failed),
+ );
+ if (!jobs.length) break;
+ const leased = new Map(jobs.map((job) => [job.userId, job]));
+ try {
+ const sources = await readLoopsProfiles(
+ cap,
+ license,
+ jobs.map((job) => job.userId),
+ ).catch((error) => {
+ console.error("Loops batch profile read failed", { error });
+ return null;
+ });
+ const now = new Date();
+ const unchanged = sources
+ ? jobs.flatMap((job) => {
+ const nextAttemptAt = unchangedUntil(
+ job,
+ sources.get(job.userId) ?? null,
+ config,
+ customerCopy,
+ now,
+ );
+ return nextAttemptAt ? [{ job, nextAttemptAt }] : [];
+ })
+ : [];
+ await completeUnchangedJobs(cap, token, unchanged);
+ for (const { job } of unchanged) leased.delete(job.userId);
+ processed += unchanged.length;
+ for (const job of leased.values()) {
+ if (Date.now() >= deadline) break;
+ leased.delete(job.userId);
+ if (!(await renewLease(cap, job, token))) continue;
+ const outcome = await syncJob(job, token);
+ if (outcome === "unchanged" || outcome === "synced") processed++;
+ else failed++;
+ if (outcome === "stop") {
+ stopped = true;
+ break;
+ }
+ if (outcome !== "unchanged")
+ await new Promise((resolve) => setTimeout(resolve, 750));
+ }
+ } finally {
+ if (leased.size)
+ await releaseJobs(cap, token, [...leased.keys()]).catch((error) =>
+ console.error("Loops lease release failed", { error }),
+ );
}
- await new Promise((resolve) => setTimeout(resolve, 750));
}
return { enabled: true, processed, failed };
} finally {
diff --git a/scripts/loops/batch.test.ts b/scripts/loops/batch.test.ts
new file mode 100644
index 000000000..c3da71940
--- /dev/null
+++ b/scripts/loops/batch.test.ts
@@ -0,0 +1,478 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import mysql, { type Connection, type RowDataPacket } from "mysql2/promise";
+import {
+ claimJobs,
+ completeUnchangedJobs,
+ releaseJobs,
+ renewLease,
+} from "../../packages/database/loops/jobs";
+import {
+ readLoopsProfile,
+ readLoopsProfiles,
+} from "../../packages/database/loops/sources";
+
+const url = process.env.LOOPS_ACTIVATION_READONLY_TEST_URL;
+
+type Row = (string | number | null)[];
+
+function fixtureRows(columns: string[], rows: Row[]) {
+ return rows.length
+ ? rows
+ .map(
+ () =>
+ `SELECT ${columns
+ .map(
+ (column) =>
+ `CAST(? AS CHAR CHARACTER SET utf8mb4) COLLATE utf8mb4_0900_ai_ci AS ${column}`,
+ )
+ .join(",")}`,
+ )
+ .join(" UNION ALL ")
+ : `SELECT ${columns.map((column) => `NULL AS ${column}`).join(",")} WHERE FALSE`;
+}
+
+const tables: Record<string, [string[], Row[]]> = {
+ users: [
+ [
+ "id",
+ "email",
+ "name",
+ "lastName",
+ "emailVerified",
+ "stripeCustomerId",
+ "stripeSubscriptionStatus",
+ "thirdPartyStripeSubscriptionId",
+ "inviteQuota",
+ "created_at",
+ "defaultOrgId",
+ "marketingOrigin",
+ ],
+ [
+ [
+ "legacy",
+ "Legacy",
+ null,
+ "2026-01-05 10:00:00",
+ "cus_legacy",
+ "active",
+ null,
+ "3",
+ "2026-01-05 10:00:00",
+ "org-legacy",
+ "independent",
+ ],
+ [
+ "recent",
+ "Recent",
+ "Person",
+ null,
+ null,
+ null,
+ null,
+ "1",
+ "2026-10-01 10:00:00",
+ "org-recent",
+ "unknown",
+ ],
+ [
+ "quiet",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ "1",
+ "2026-10-02 10:00:00",
+ null,
+ "unknown",
+ ],
+ [
+ "mixedcase",
+ "Mixed",
+ null,
+ "2026-03-01 10:00:00",
+ null,
+ "canceled",
+ null,
+ "1",
+ "2026-03-01 10:00:00",
+ null,
+ "unknown",
+ ],
+ [
+ "accent",
+ "José@example.com",
+ "José",
+ null,
+ null,
+ null,
+ null,
+ "sub_partner",
+ "1",
+ "2026-02-01 10:00:00",
+ null,
+ "teammate",
+ ],
+ ],
+ ],
+ accounts: [
+ ["userId", "provider"],
+ [
+ ["legacy", "google"],
+ ["accent", "workos"],
+ ],
+ ],
+ organization_invites: [
+ ["invitedEmail", "status"],
+ [
+ ["[email protected]", "pending"],
+ ["[email protected]", "accepted"],
+ ],
+ ],
+ organization_members: [
+ ["userId", "organizationId", "hasProSeat"],
+ [
+ ["legacy", "org-legacy", "0"],
+ ["legacy", "org-shared", "1"],
+ ["MIXEDCASE", "org-shared", "0"],
+ ["recent", "org-recent", "0"],
+ ],
+ ],
+ organizations: [
+ ["id", "ownerId", "tombstoneAt", "workosConnectionId", "createdAt"],
+ [
+ ["org-legacy", "legacy", null, null, "2026-01-05 10:00:00"],
+ ["org-shared", "someone", "2026-06-01 10:00:00", "conn", "2026-01-01"],
+ ["org-recent", "recent", null, null, "2026-10-01 10:00:00"],
+ ],
+ ],
+ videos: [
+ [
+ "id",
+ "ownerId",
+ "isScreenshot",
+ "duration",
+ "source",
+ "jobStatus",
+ "createdAt",
+ "firstViewEmailSentAt",
+ "public",
+ ],
+ [
+ [
+ "v-legacy-private",
+ "legacy",
+ "0",
+ "12",
+ '{"type":"MediaConvert"}',
+ null,
+ "2026-01-06 10:00:00",
+ null,
+ "0",
+ ],
+ [
+ "v-legacy-public",
+ "legacy",
+ "0",
+ "30",
+ '{"type":"MediaConvert"}',
+ "COMPLETE",
+ "2026-01-07 10:00:00",
+ null,
+ "1",
+ ],
+ [
+ "v-recent-shared",
+ "recent",
+ "0",
+ "45",
+ '{"type":"desktopMP4","outputKey":"recent/v/result.mp4"}',
+ null,
+ "2026-10-02 10:00:00",
+ "2026-10-02 12:00:00",
+ "1",
+ ],
+ [
+ "v-recent-uploading",
+ "recent",
+ "0",
+ "20",
+ '{"type":"webMP4"}',
+ null,
+ "2026-10-03 10:00:00",
+ null,
+ "1",
+ ],
+ [
+ "v-recent-shot",
+ "recent",
+ "1",
+ "0",
+ '{"type":"desktopMP4"}',
+ null,
+ "2026-10-04 10:00:00",
+ null,
+ "1",
+ ],
+ [
+ "v-mixed",
+ "MIXEDCASE",
+ "0",
+ "9",
+ '{"type":"desktopSegments"}',
+ null,
+ "2026-03-02 10:00:00",
+ null,
+ "1",
+ ],
+ ],
+ ],
+ video_uploads: [["video_id"], [["v-recent-uploading"]]],
+ video_processing_jobs: [
+ ["video_id", "state"],
+ [["v-recent-shared", "verified"]],
+ ],
+ user: [
+ ["id", "email"],
+ [
+ ["license-legacy", "[email protected]"],
+ ["license-recent", "[email protected]"],
+ ["license-accent", "[email protected]"],
+ ],
+ ],
+ commercialLicenses: [
+ ["userId", "subscriptionActive", "nextRenewalDate"],
+ [
+ ["license-legacy", "1", "2030-01-01 00:00:00"],
+ ["license-recent", "0", null],
+ ["license-accent", "1", "2020-01-01 00:00:00"],
+ ],
+ ],
+ selfHostedLicenses: [
+ ["userId", "subscriptionActive", "nextRenewalDate"],
+ [["license-recent", "1", "2030-01-01 00:00:00"]],
+ ],
+};
+
+const cte = `WITH ${Object.entries(tables)
+ .map(
+ ([name, [columns, rows]]) => `${name} AS (${fixtureRows(columns, rows)})`,
+ )
+ .join(", ")}`;
+const cteValues = Object.values(tables).flatMap(([, rows]) => rows.flat());
+
+describe.skipIf(!url)("batched Loops source reads", () => {
+ let database: Connection;
+ let fixtures: Connection;
+ const statements: string[] = [];
+ beforeAll(async () => {
+ if (!url) throw new Error("Missing read-only test connection");
+ database = await mysql.createConnection({
+ uri: url,
+ dateStrings: true,
+ timezone: "Z",
+ });
+ fixtures = {
+ execute: (sql: string, values: unknown[]) => {
+ statements.push(sql);
+ return database.query(`${cte} ${sql}`, [...cteValues, ...values]);
+ },
+ } as unknown as Connection;
+ });
+ afterAll(async () => {
+ await database?.end();
+ });
+
+ const withoutClock = (
+ profile: Awaited<ReturnType<typeof readLoopsProfile>>,
+ ) =>
+ profile && {
+ ...profile,
+ input: { ...profile.input, now: undefined },
+ };
+
+ test("every user reads the same profile as the single-user path", async () => {
+ const requested = [
+ "legacy",
+ "recent",
+ "quiet",
+ "MixedCase",
+ "accent",
+ "missing",
+ ];
+ const batch = await readLoopsProfiles(fixtures, fixtures, requested);
+ for (const userId of requested)
+ expect(withoutClock(batch.get(userId) ?? null)).toEqual(
+ withoutClock(await readLoopsProfile(fixtures, fixtures, userId)),
+ );
+ expect([...batch.keys()].sort()).toEqual(
+ requested.filter((userId) => userId !== "missing").sort(),
+ );
+ });
+
+ test("the fixtures exercise every source", async () => {
+ const batch = await readLoopsProfiles(fixtures, fixtures, [
+ "legacy",
+ "recent",
+ "quiet",
+ "MixedCase",
+ "accent",
+ ]);
+ const legacy = batch.get("legacy")?.input;
+ expect(legacy?.memberships.map((member) => member.organizationId)).toEqual([
+ "org-legacy",
+ "org-shared",
+ ]);
+ expect([legacy?.hasVideo, legacy?.hasSharedVideo]).toEqual([true, true]);
+ expect(legacy?.licenses.map((license) => license.kind)).toEqual([
+ "desktop",
+ ]);
+ const recent = batch.get("recent")?.input;
+ expect(recent?.licenses.map((license) => license.kind)).toEqual([
+ "desktop",
+ "selfhosted",
+ ]);
+ expect([
+ recent?.hasVideo,
+ recent?.hasSharedVideo,
+ recent?.hasPendingUpload,
+ ]).toEqual([true, true, true]);
+ expect(recent?.lastActivationNotificationAt).toBe("2026-10-03 10:05:00");
+ const quiet = batch.get("quiet");
+ expect(quiet?.pendingInvite).toBe(true);
+ expect(quiet?.input.lastActivationNotificationAt).toBeNull();
+ expect(batch.get("MixedCase")?.input.memberships).toHaveLength(1);
+ expect(batch.get("MixedCase")?.input.hasVideo).toBe(true);
+ expect(batch.get("accent")?.input.licenses).toHaveLength(1);
+ expect(batch.get("accent")?.input.sso).toBe(true);
+ });
+
+ test("a batch costs one statement per source table", async () => {
+ statements.length = 0;
+ await readLoopsProfiles(fixtures, fixtures, [
+ "legacy",
+ "recent",
+ "quiet",
+ "MixedCase",
+ "accent",
+ ]);
+ expect(statements).toHaveLength(5);
+ statements.length = 0;
+ await readLoopsProfiles(fixtures, fixtures, ["legacy", "MixedCase"]);
+ expect(statements).toHaveLength(4);
+ statements.length = 0;
+ expect(
+ (await readLoopsProfiles(fixtures, fixtures, ["missing"])).size,
+ ).toBe(0);
+ expect(statements).toHaveLength(1);
+ });
+});
+
+describe.skipIf(!url)("batched Loops job leases", () => {
+ let database: Connection;
+ beforeAll(async () => {
+ if (!url) throw new Error("Missing read-only test connection");
+ database = await mysql.createConnection({
+ uri: url,
+ dateStrings: true,
+ timezone: "Z",
+ });
+ await database.query(
+ "CREATE TEMPORARY TABLE loops_sync_jobs (userId varchar(15) NOT NULL, revision int NOT NULL DEFAULT 1, nextAttemptAt datetime NOT NULL, leaseToken varchar(36) DEFAULT NULL, leaseUntil datetime DEFAULT NULL, failures int NOT NULL DEFAULT 0, lastError varchar(64) DEFAULT NULL, profileHash varchar(64) DEFAULT NULL, syncedEmail varchar(255) DEFAULT NULL, lastSyncedAt datetime DEFAULT NULL, teammateJoinedAt datetime DEFAULT NULL, PRIMARY KEY (userId), KEY loops_sync_due_idx (nextAttemptAt))",
+ );
+ await database.query(
+ `INSERT INTO loops_sync_jobs (userId,nextAttemptAt,leaseToken,leaseUntil,failures,lastError) VALUES
+ ('failing',UTC_TIMESTAMP()-INTERVAL 4 MINUTE,NULL,NULL,3,'loops_http_500'),
+ ('first',UTC_TIMESTAMP()-INTERVAL 3 MINUTE,NULL,NULL,0,NULL),
+ ('second',UTC_TIMESTAMP()-INTERVAL 2 MINUTE,NULL,NULL,0,NULL),
+ ('third',UTC_TIMESTAMP()-INTERVAL 1 MINUTE,NULL,NULL,0,NULL),
+ ('expired',UTC_TIMESTAMP()-INTERVAL 30 SECOND,'old',UTC_TIMESTAMP()-INTERVAL 1 SECOND,0,NULL),
+ ('leased',UTC_TIMESTAMP()-INTERVAL 5 MINUTE,'other',UTC_TIMESTAMP()+INTERVAL 1 MINUTE,0,NULL),
+ ('future',UTC_TIMESTAMP()+INTERVAL 1 HOUR,NULL,NULL,0,NULL)`,
+ );
+ });
+ afterAll(async () => {
+ await database?.end();
+ });
+
+ const job = async (userId: string) => {
+ const [[row]] = await database.query<RowDataPacket[]>(
+ "SELECT *, leaseUntil>UTC_TIMESTAMP() AS leaseActive, TIMESTAMPDIFF(SECOND,UTC_TIMESTAMP(),nextAttemptAt) AS dueIn FROM loops_sync_jobs WHERE userId=?",
+ [userId],
+ );
+ return row;
+ };
+
+ test("claims due jobs in order and leases them together", async () => {
+ const jobs = await claimJobs(database, "batch", null, 3);
+ expect(jobs.map((claimed) => claimed.userId)).toEqual([
+ "failing",
+ "first",
+ "second",
+ ]);
+ for (const userId of ["failing", "first", "second"])
+ expect(await job(userId)).toMatchObject({
+ leaseToken: "batch",
+ leaseActive: 1,
+ });
+ expect(
+ (await claimJobs(database, "rest", null, 50)).map(
+ (claimed) => claimed.userId,
+ ),
+ ).toEqual(["third", "expired"]);
+ expect(await claimJobs(database, "none", null, 50)).toEqual([]);
+ });
+
+ test("completes unchanged jobs with the revision guard", async () => {
+ await database.query(
+ "UPDATE loops_sync_jobs SET revision=revision+1 WHERE userId='first'",
+ );
+ const wholeSecond = Math.floor(Date.now() / 1000) * 1000;
+ const day = new Date(wholeSecond + 24 * 60 * 60_000);
+ const hour = new Date(wholeSecond + 60 * 60_000);
+ await completeUnchangedJobs(database, "batch", [
+ { job: { userId: "failing", revision: 1 }, nextAttemptAt: day },
+ { job: { userId: "first", revision: 1 }, nextAttemptAt: day },
+ { job: { userId: "second", revision: 1 }, nextAttemptAt: hour },
+ { job: { userId: "third", revision: 1 }, nextAttemptAt: hour },
+ ]);
+ expect(await job("failing")).toMatchObject({
+ failures: 0,
+ lastError: null,
+ leaseToken: null,
+ leaseUntil: null,
+ });
+ expect((await job("failing")).dueIn).toBeGreaterThan(23 * 60 * 60);
+ expect((await job("first")).dueIn).toBeLessThanOrEqual(0);
+ expect((await job("first")).leaseToken).toBeNull();
+ expect((await job("second")).dueIn).toBeGreaterThan(59 * 60);
+ expect((await job("second")).dueIn).toBeLessThanOrEqual(60 * 60);
+ expect(await job("third")).toMatchObject({ leaseToken: "rest" });
+ expect((await job("third")).dueIn).toBeLessThanOrEqual(0);
+ });
+
+ test("renews and releases only leases it holds", async () => {
+ expect(await renewLease(database, { userId: "third" }, "rest")).toBe(true);
+ expect(await renewLease(database, { userId: "third" }, "rest")).toBe(true);
+ expect(await renewLease(database, { userId: "third" }, "batch")).toBe(
+ false,
+ );
+ await releaseJobs(database, "batch", ["third", "expired"]);
+ expect((await job("third")).leaseToken).toBe("rest");
+ await releaseJobs(database, "rest", ["third", "expired"]);
+ expect(await job("third")).toMatchObject({
+ leaseToken: null,
+ leaseUntil: null,
+ });
+ expect(
+ (await claimJobs(database, "again", ["expired", "future"], 50)).map(
+ (claimed) => claimed.userId,
+ ),
+ ).toEqual(["expired"]);
+ });
+});
diff --git a/scripts/loops/database.test.ts b/scripts/loops/database.test.ts
index 30a5f5632..39c5360dc 100644
--- a/scripts/loops/database.test.ts
+++ b/scripts/loops/database.test.ts
@@ -5,15 +5,20 @@ import { eq } from "drizzle-orm";
import { drizzle } from "drizzle-orm/mysql2";
import mysql, { type Connection, type RowDataPacket } from "mysql2/promise";
import { readQueueHealth } from "../../packages/database/loops/health";
-import { enqueueLoopsSync } from "../../packages/database/loops/queue";
import {
- claimJob,
+ claimJobs,
completeUnchangedJob,
deferJob,
-} from "../../packages/database/loops/worker";
+} from "../../packages/database/loops/jobs";
+import { enqueueLoopsSync } from "../../packages/database/loops/queue";
import { loopsSyncJobs } from "../../packages/database/schema";
const url = process.env.LOOPS_TEST_DATABASE_URL;
+const claimJob = async (
+ database: Connection,
+ token: string,
+ allowedUserIds: string[],
+) => (await claimJobs(database, token, allowedUserIds, 1))[0];
const ids = Array.from({ length: 7 }, () =>
User.UserId.make(randomUUID().replaceAll("-", "").slice(0, 15)),
);