Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Make partner seed script deterministic & chunked

Rafi98ca0fe9be0a
diff --git a/apps/web/scripts/dev/seed-100k-partners.ts b/apps/web/scripts/dev/seed-100k-partners.ts
index b382d1ab01..108bb0d2a9 100644
--- a/apps/web/scripts/dev/seed-100k-partners.ts
+++ b/apps/web/scripts/dev/seed-100k-partners.ts
@@ -1,34 +1,67 @@
-import { createId } from "@/lib/api/create-id";
import { hashPassword } from "@/lib/auth/password";
import { prisma } from "@/lib/prisma";
import { PlatformType, Prisma } from "@prisma/client";
+import { createHash } from "crypto";
import "dotenv-flow/config";
+const DEFAULT_COUNT = 100_000;
+const DEFAULT_SEED = "partners-search";
+const CHUNK_SIZE = 2_500;
+
+const createDeterministicId = ({
+ namespace,
+ prefix,
+ index,
+}: {
+ namespace: string;
+ prefix: string;
+ index: number;
+}) =>
+ `${prefix}${createHash("sha256")
+ .update(`${namespace}:${prefix}:${index}`)
+ .digest("hex")
+ .slice(0, 24)}`;
+
+const parsePositiveInteger = (value: string | undefined, flag: string) => {
+ const parsed = Number(value);
+
+ if (!Number.isSafeInteger(parsed) || parsed <= 0) {
+ throw new Error(`${flag} must be a positive integer.`);
+ }
+
+ return parsed;
+};
+
// Dataset arrays for diverse generation
+// prettier-ignore
const FIRST_NAMES = [
"Alex", "Jordan", "Taylor", "Morgan", "Chris", "Sam", "Riley", "Casey", "Dakota", "Jamie",
"Avery", "Reese", "Skyler", "Quinn", "Rowan", "Peyton", "Finley", "Emerson", "Hayden", "Sage",
"Logan", "Jesse", "Harper", "Eden", "Kendall", "Devon", "Dallas", "Shiloh", "River", "Phoenix",
- "Cameron", "Drew", "Eli", "Francis", "Greyson", "Hadley", "Jules", "Kai", "Lennon", "Marlowe"
+ "Cameron", "Drew", "Eli", "Francis", "Greyson", "Hadley", "Jules", "Kai", "Lennon", "Marlowe",
];
+// prettier-ignore
const LAST_NAMES = [
"Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Rodriguez", "Martinez",
"Hernandez", "Lopez", "Gonzalez", "Wilson", "Anderson", "Thomas", "Taylor", "Moore", "Jackson", "Martin",
"Lee", "Perez", "Thompson", "White", "Harris", "Sanchez", "Clark", "Ramirez", "Lewis", "Robinson",
- "Walker", "Young", "Allen", "King", "Wright", "Scott", "Torres", "Nguyen", "Hill", "Flores"
+ "Walker", "Young", "Allen", "King", "Wright", "Scott", "Torres", "Nguyen", "Hill", "Flores",
];
+// prettier-ignore
const COMPANY_SUFFIXES = [
"Tech", "Labs", "Media", "Agency", "Studio", "Ventures", "Digital", "Creative", "Global", "Solutions",
- "Interactive", "Capital", "Partners", "Holdings", "Group", "Network", "Cloud", "AI", "Analytics", "Growth"
+ "Interactive", "Capital", "Partners", "Holdings", "Group", "Network", "Cloud", "AI", "Analytics", "Growth",
];
+// prettier-ignore
const DOMAINS = [
"example.com", "techcorp.io", "marketing.co", "acme.dev", "growth.app",
- "agency.net", "saas.com", "creator.xyz", "dub.co", "builder.build"
+ "agency.net", "saas.com", "creator.xyz", "dub.co", "builder.build",
];
+// prettier-ignore
const COUNTRIES = ["US", "CA", "GB", "DE", "FR", "AU", "JP", "IN", "BR", "NL", "ES", "SE", "SG"];
const DESCRIPTIONS = [
@@ -39,7 +72,7 @@ const DESCRIPTIONS = [
"Social media influencer creating tech reviews, tutorials, and unboxings.",
"E-commerce consultant helping brands scale through affiliate networks.",
"Newsletter creator focused on modern web development and AI tools.",
- "Community leader running a developer network and podcast."
+ "Community leader running a developer network and podcast.",
];
const PLATFORM_TYPES: PlatformType[] = [
@@ -48,34 +81,291 @@ const PLATFORM_TYPES: PlatformType[] = [
PlatformType.twitter,
PlatformType.linkedin,
PlatformType.instagram,
- PlatformType.tiktok
+ PlatformType.tiktok,
];
-async function main() {
- const args = process.argv.slice(2);
- let totalCount = 100000;
+type SeedArguments = {
+ totalCount: number;
+ targetProgramId: string | null;
+ seed: string;
+};
+
+type PartnerChunk = {
+ users: Prisma.UserCreateManyInput[];
+ partners: Prisma.PartnerCreateManyInput[];
+ partnerUsers: Prisma.PartnerUserCreateManyInput[];
+ enrollments: Prisma.ProgramEnrollmentCreateManyInput[];
+ platforms: Prisma.PartnerPlatformCreateManyInput[];
+ links: Prisma.LinkCreateManyInput[];
+};
+
+type GeneratePartnerChunkOptions = {
+ start: number;
+ end: number;
+ seedNamespace: string;
+ seedFingerprint: string;
+ passwordHash: string;
+ runStartedAt: Date;
+ programId: string;
+ defaultGroupId: string | null;
+ programDomain: string | null;
+ workspaceId: string;
+};
+
+// These helpers are intentionally file-local; this script is their only caller.
+const parseArguments = (args: string[]): SeedArguments => {
+ let totalCount = DEFAULT_COUNT;
let targetProgramId: string | null = null;
+ let seed = DEFAULT_SEED;
for (const arg of args) {
if (arg.startsWith("--count=")) {
- totalCount = parseInt(arg.split("=")[1], 10) || 100000;
+ totalCount = parsePositiveInteger(arg.split("=")[1], "--count");
} else if (arg.startsWith("--programId=")) {
targetProgramId = arg.split("=")[1];
+ } else if (arg.startsWith("--seed=")) {
+ seed = arg.split("=")[1];
+ } else {
+ throw new Error(`Unknown argument: ${arg}`);
+ }
+ }
+
+ if (!seed || !/^[a-zA-Z0-9_-]{1,40}$/.test(seed)) {
+ throw new Error(
+ "--seed must contain 1-40 letters, numbers, underscores, or hyphens.",
+ );
+ }
+
+ if (targetProgramId === "") {
+ throw new Error("--programId cannot be empty.");
+ }
+
+ return { totalCount, targetProgramId, seed };
+};
+
+const resolveProgramId = async (targetProgramId: string | null) => {
+ if (targetProgramId) {
+ return targetProgramId;
+ }
+
+ const programs = await prisma.program.findMany({
+ take: 2,
+ select: { id: true, name: true },
+ });
+
+ if (programs.length > 1) {
+ throw new Error(
+ `Multiple programs found. Pass --programId=<id> to choose one (for example, ${programs[0].id} for "${programs[0].name}").`,
+ );
+ }
+
+ return programs[0]?.id ?? null;
+};
+
+const generatePartnerChunk = ({
+ start,
+ end,
+ seedNamespace,
+ seedFingerprint,
+ passwordHash,
+ runStartedAt,
+ programId,
+ defaultGroupId,
+ programDomain,
+ workspaceId,
+}: GeneratePartnerChunkOptions): PartnerChunk => {
+ const users: Prisma.UserCreateManyInput[] = [];
+ const partners: Prisma.PartnerCreateManyInput[] = [];
+ const partnerUsers: Prisma.PartnerUserCreateManyInput[] = [];
+ const enrollments: Prisma.ProgramEnrollmentCreateManyInput[] = [];
+ const platforms: Prisma.PartnerPlatformCreateManyInput[] = [];
+ const links: Prisma.LinkCreateManyInput[] = [];
+
+ for (let i = start; i < end; i++) {
+ const partnerId = createDeterministicId({
+ namespace: seedNamespace,
+ prefix: "pn_",
+ index: i,
+ });
+ const userId = createDeterministicId({
+ namespace: seedNamespace,
+ prefix: "user_",
+ index: i,
+ });
+ const enrollmentId = createDeterministicId({
+ namespace: seedNamespace,
+ prefix: "pge_",
+ index: i,
+ });
+
+ const firstName = FIRST_NAMES[i % FIRST_NAMES.length];
+ const lastName =
+ LAST_NAMES[(i + Math.floor(i / FIRST_NAMES.length)) % LAST_NAMES.length];
+ const name = `${firstName} ${lastName}`;
+
+ // Include deliberate prefix and infix cases for search verification.
+ let emailPrefix: string;
+ if (i % 100 === 0) {
+ emailPrefix = `partner.${seedFingerprint}.${i}`;
+ } else if (i % 137 === 0) {
+ emailPrefix = `substringneedle.${seedFingerprint}.${i}`;
+ } else if (i % 75 === 0) {
+ emailPrefix = `tech.creator.${seedFingerprint}.${i}`;
+ } else if (i % 50 === 0) {
+ emailPrefix = `dub.affiliate.${seedFingerprint}.${i}`;
+ } else {
+ emailPrefix = `${firstName.toLowerCase()}.${lastName.toLowerCase()}.${seedFingerprint}.${i}`;
+ }
+
+ const domain = DOMAINS[i % DOMAINS.length];
+ const email = `${emailPrefix}@${domain}`;
+ const username = `${firstName.toLowerCase()}_${lastName.toLowerCase()}_${seedFingerprint}_${i}`;
+ const companyName = `${lastName} ${COMPANY_SUFFIXES[i % COMPANY_SUFFIXES.length]}`;
+ const country = COUNTRIES[i % COUNTRIES.length];
+ const description = DESCRIPTIONS[i % DESCRIPTIONS.length];
+ const createdAt = new Date(
+ runStartedAt.getTime() - ((i * 60_000) % (365 * 86_400_000)),
+ );
+
+ users.push({
+ id: userId,
+ name,
+ email,
+ emailVerified: runStartedAt,
+ passwordHash,
+ defaultPartnerId: partnerId,
+ createdAt,
+ });
+
+ partners.push({
+ id: partnerId,
+ name,
+ username,
+ email,
+ description,
+ country,
+ companyName,
+ networkStatus: "approved",
+ createdAt,
+ });
+
+ partnerUsers.push({
+ id: createDeterministicId({
+ namespace: seedNamespace,
+ prefix: "pnusr_",
+ index: i,
+ }),
+ userId,
+ partnerId,
+ role: "owner",
+ createdAt,
+ });
+
+ enrollments.push({
+ id: enrollmentId,
+ partnerId,
+ programId,
+ groupId: defaultGroupId,
+ status: "approved",
+ createdAt,
+ });
+
+ // Give every partner one or two records across all supported platforms.
+ const numPlatforms = 1 + (i % 2);
+ for (let p = 0; p < numPlatforms; p++) {
+ const platformType = PLATFORM_TYPES[(i + p) % PLATFORM_TYPES.length];
+ const identifier =
+ platformType === PlatformType.website
+ ? `https://www.${companyName.toLowerCase().replace(/[^a-z0-9]/g, "")}-${i}.${domain}`
+ : `@${firstName.toLowerCase()}_${lastName.toLowerCase()}_${i}`;
+
+ platforms.push({
+ id: createDeterministicId({
+ namespace: seedNamespace,
+ prefix: `pnp${p}_`,
+ index: i,
+ }),
+ partnerId,
+ type: platformType,
+ identifier,
+ subscribers: BigInt(100 + ((i * 37) % 50000)),
+ views: BigInt(500 + ((i * 123) % 500000)),
+ verifiedAt: createdAt,
+ createdAt,
+ });
}
+
+ // Give every partner one searchable referral link.
+ const linkKey = `p-${seedFingerprint}-${i}`;
+ const linkDomain = programDomain || "dub.sh";
+ links.push({
+ id: createDeterministicId({
+ namespace: seedNamespace,
+ prefix: "link_",
+ index: i,
+ }),
+ domain: linkDomain,
+ key: linkKey,
+ url: `https://${domain}/ref/${username}`,
+ shortLink: `https://${linkDomain}/${linkKey}`,
+ projectId: workspaceId,
+ programId,
+ partnerId,
+ createdAt,
+ });
}
- console.log(`\nπŸš€ Starting 100K+ Partner Data Seed (Target Count: ${totalCount.toLocaleString()})...`);
+ return { users, partners, partnerUsers, enrollments, platforms, links };
+};
+
+const insertPartnerChunk = async ({
+ users,
+ partners,
+ partnerUsers,
+ enrollments,
+ platforms,
+ links,
+}: PartnerChunk) => {
+ // Keep every chunk atomic. Stable IDs make the same seed safe to resume/rerun.
+ const [, partnerResult] = await prisma.$transaction([
+ prisma.user.createMany({ data: users, skipDuplicates: true }),
+ prisma.partner.createMany({ data: partners, skipDuplicates: true }),
+ prisma.partnerUser.createMany({ data: partnerUsers, skipDuplicates: true }),
+ prisma.programEnrollment.createMany({
+ data: enrollments,
+ skipDuplicates: true,
+ }),
+ prisma.partnerPlatform.createMany({
+ data: platforms,
+ skipDuplicates: true,
+ }),
+ prisma.link.createMany({ data: links, skipDuplicates: true }),
+ ]);
+
+ return partnerResult.count;
+};
+
+async function main() {
+ // Step 1: Parse and validate command-line options.
+ const { totalCount, targetProgramId, seed } = parseArguments(
+ process.argv.slice(2),
+ );
+
+ // Step 2: Resolve the target program and its workspace.
+ const resolvedProgramId = await resolveProgramId(targetProgramId);
- // Find target program & workspace
- const program = targetProgramId
- ? await prisma.program.findUnique({ where: { id: targetProgramId } })
- : await prisma.program.findFirst();
+ console.log(
+ `\nπŸš€ Starting Partner Data Seed (Target Count: ${totalCount.toLocaleString()}, Seed: "${seed}")...`,
+ );
+
+ const program = resolvedProgramId
+ ? await prisma.program.findUnique({ where: { id: resolvedProgramId } })
+ : null;
if (!program) {
- console.error(
- "❌ No program found in database. Please run 'pnpm run script dev/seed' first to set up the default workspace and program."
+ throw new Error(
+ "❌ No program found in database. Please run 'pnpm run script dev/seed' first to set up the default workspace and program.",
);
- process.exit(1);
}
const workspace = await prisma.project.findUnique({
@@ -83,157 +373,58 @@ async function main() {
});
if (!workspace) {
- console.error("❌ Program workspace not found.");
- process.exit(1);
+ throw new Error("❌ Program workspace not found.");
}
- console.log(`πŸ“ Seeding partners for Program: "${program.name}" (${program.id})`);
+ console.log(
+ `πŸ“ Seeding partners for Program: "${program.name}" (${program.id})`,
+ );
console.log(` Workspace: "${workspace.name}" (${workspace.id})\n`);
+ // Step 3: Build the stable namespace shared by every generated chunk.
const passwordHash = await hashPassword("password");
- const chunkSize = 2500;
- const totalChunks = Math.ceil(totalCount / chunkSize);
+ const seedNamespace = `${program.id}:${seed}`;
+ const seedFingerprint = createHash("sha256")
+ .update(seedNamespace)
+ .digest("hex")
+ .slice(0, 10);
+ const runStartedAt = new Date();
+ const totalChunks = Math.ceil(totalCount / CHUNK_SIZE);
const startTime = Date.now();
+ let insertedPartners = 0;
+ // Step 4: Generate and atomically insert one bounded chunk at a time.
for (let chunk = 0; chunk < totalChunks; chunk++) {
- const chunkStart = chunk * chunkSize;
- const chunkEnd = Math.min(chunkStart + chunkSize, totalCount);
- const currentChunkSize = chunkEnd - chunkStart;
-
- const chunkUsers: Prisma.UserCreateManyInput[] = [];
- const chunkPartners: Prisma.PartnerCreateManyInput[] = [];
- const chunkPartnerUsers: Prisma.PartnerUserCreateManyInput[] = [];
- const chunkEnrollments: Prisma.ProgramEnrollmentCreateManyInput[] = [];
- const chunkPlatforms: Prisma.PartnerPlatformCreateManyInput[] = [];
- const chunkLinks: Prisma.LinkCreateManyInput[] = [];
-
- for (let i = chunkStart; i < chunkEnd; i++) {
- const partnerId = createId({ prefix: "pn_" });
- const userId = createId({ prefix: "user_" });
- const enrollmentId = createId({ prefix: "pge_" });
-
- const firstName = FIRST_NAMES[i % FIRST_NAMES.length];
- const lastName = LAST_NAMES[(i + Math.floor(i / FIRST_NAMES.length)) % LAST_NAMES.length];
- const name = `${firstName} ${lastName}`;
-
- const runSuffix = Math.floor(Date.now() / 1000).toString(36);
- // Email generation with deliberate test cases for search verification
- let emailPrefix: string;
- if (i % 100 === 0) {
- emailPrefix = `examp.partner.${runSuffix}.${i}`; // Ensures partial match 'examp' works
- } else if (i % 75 === 0) {
- emailPrefix = `tech.creator.${runSuffix}.${i}`;
- } else if (i % 50 === 0) {
- emailPrefix = `dub.affiliate.${runSuffix}.${i}`;
- } else {
- emailPrefix = `${firstName.toLowerCase()}.${lastName.toLowerCase()}.${runSuffix}.${i}`;
- }
-
- const domain = DOMAINS[i % DOMAINS.length];
- const email = `${emailPrefix}@${domain}`;
- const username = `${firstName.toLowerCase()}_${lastName.toLowerCase()}_${runSuffix}_${i}`;
- const companyName = `${lastName} ${COMPANY_SUFFIXES[i % COMPANY_SUFFIXES.length]}`;
- const country = COUNTRIES[i % COUNTRIES.length];
- const description = DESCRIPTIONS[i % DESCRIPTIONS.length];
- const createdAt = new Date(Date.now() - (i * 60000) % (365 * 86400000));
-
- // User record
- chunkUsers.push({
- id: userId,
- name,
- email,
- emailVerified: new Date(),
- passwordHash,
- defaultPartnerId: partnerId,
- createdAt,
- });
-
- // Partner record
- chunkPartners.push({
- id: partnerId,
- name,
- username,
- email,
- description,
- country,
- companyName,
- networkStatus: "approved",
- createdAt,
- });
-
- // PartnerUser record
- chunkPartnerUsers.push({
- id: createId({ prefix: "pn_" }), // or unique ID
- userId,
- partnerId,
- role: "owner",
- createdAt,
- });
-
- // Program Enrollment record
- chunkEnrollments.push({
- id: enrollmentId,
- partnerId,
- programId: program.id,
- groupId: program.defaultGroupId,
- status: "approved",
- createdAt,
- });
-
- // Generate 1-2 Platforms per partner
- const numPlatforms = 1 + (i % 2);
- for (let p = 0; p < numPlatforms; p++) {
- const platformType = PLATFORM_TYPES[(i + p) % PLATFORM_TYPES.length];
- const identifier =
- platformType === PlatformType.website
- ? `https://www.${companyName.toLowerCase().replace(/[^a-z0-9]/g, "")}-${i}.${domain}`
- : `@${firstName.toLowerCase()}_${lastName.toLowerCase()}_${i}`;
-
- chunkPlatforms.push({
- id: createId({ prefix: "pn_" }),
- partnerId,
- type: platformType,
- identifier,
- subscribers: BigInt(100 + ((i * 37) % 50000)),
- views: BigInt(500 + ((i * 123) % 500000)),
- verifiedAt: createdAt,
- createdAt,
- });
- }
-
- // Generate 1 Short Link per partner
- const linkKey = `p-${firstName.toLowerCase()}-${lastName.toLowerCase()}-${i}`;
- const linkDomain = program.domain || "dub.sh";
- chunkLinks.push({
- id: createId({ prefix: "link_" }),
- domain: linkDomain,
- key: linkKey,
- url: `https://${domain}/ref/${username}`,
- shortLink: `https://${linkDomain}/${linkKey}`,
- projectId: workspace.id,
- programId: program.id,
- partnerId,
- createdAt,
- });
- }
-
- // Perform bulk insertions
- await prisma.user.createMany({ data: chunkUsers, skipDuplicates: true });
- await prisma.partner.createMany({ data: chunkPartners, skipDuplicates: true });
- await prisma.partnerUser.createMany({ data: chunkPartnerUsers, skipDuplicates: true });
- await prisma.programEnrollment.createMany({ data: chunkEnrollments, skipDuplicates: true });
- await prisma.partnerPlatform.createMany({ data: chunkPlatforms, skipDuplicates: true });
- await prisma.link.createMany({ data: chunkLinks, skipDuplicates: true });
-
+ const chunkStart = chunk * CHUNK_SIZE;
+ const chunkEnd = Math.min(chunkStart + CHUNK_SIZE, totalCount);
+ const partnerChunk = generatePartnerChunk({
+ start: chunkStart,
+ end: chunkEnd,
+ seedNamespace,
+ seedFingerprint,
+ passwordHash,
+ runStartedAt,
+ programId: program.id,
+ defaultGroupId: program.defaultGroupId,
+ programDomain: program.domain,
+ workspaceId: workspace.id,
+ });
+ const insertedInChunk = await insertPartnerChunk(partnerChunk);
+ insertedPartners += insertedInChunk;
+
+ // Report both processing progress and the idempotent insert count.
const elapsedSec = ((Date.now() - startTime) / 1000).toFixed(1);
const progressPct = (((chunk + 1) / totalChunks) * 100).toFixed(0);
console.log(
- ` [Chunk ${chunk + 1}/${totalChunks}] (${progressPct}%) Inserted ${chunkEnd.toLocaleString()}/${totalCount.toLocaleString()} partners... (${elapsedSec}s elapsed)`
+ ` [Chunk ${chunk + 1}/${totalChunks}] (${progressPct}%) Processed ${chunkEnd.toLocaleString()}/${totalCount.toLocaleString()} partners (${insertedInChunk.toLocaleString()} new)... (${elapsedSec}s elapsed)`,
);
}
+ // Step 5: Summarize how much of the requested seed was newly inserted.
const totalTimeSec = ((Date.now() - startTime) / 1000).toFixed(1);
- console.log(`\nβœ… Successfully seeded ${totalCount.toLocaleString()} partners in ${totalTimeSec} seconds!`);
+ console.log(
+ `\nβœ… Seed complete: ${insertedPartners.toLocaleString()} new, ${(totalCount - insertedPartners).toLocaleString()} already present (${totalTimeSec}s).`,
+ );
}
main()