The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
Make partner seed script deterministic & chunked
Rafi98ca0fe9be0a
diff --git a/apps/web/scripts/dev/seed-100k-partners.ts b/apps/web/scripts/dev/seed-100k-partners.tsindex b382d1ab01..108bb0d2a9 100644--- a/apps/web/scripts/dev/seed-100k-partners.ts+++ b/apps/web/scripts/dev/seed-100k-partners.ts@@ -1,34 +1,67 @@-import { createId } from "@/lib/api/create-id";import { hashPassword } from "@/lib/auth/password";import { prisma } from "@/lib/prisma";import { PlatformType, Prisma } from "@prisma/client";+import { createHash } from "crypto";import "dotenv-flow/config";+const DEFAULT_COUNT = 100_000;+const DEFAULT_SEED = "partners-search";+const CHUNK_SIZE = 2_500;++const createDeterministicId = ({+ namespace,+ prefix,+ index,+}: {+ namespace: string;+ prefix: string;+ index: number;+}) =>+ `${prefix}${createHash("sha256")+ .update(`${namespace}:${prefix}:${index}`)+ .digest("hex")+ .slice(0, 24)}`;++const parsePositiveInteger = (value: string | undefined, flag: string) => {+ const parsed = Number(value);++ if (!Number.isSafeInteger(parsed) || parsed <= 0) {+ throw new Error(`${flag} must be a positive integer.`);+ }++ return parsed;+};+// Dataset arrays for diverse generation+// prettier-ignoreconst FIRST_NAMES = ["Alex", "Jordan", "Taylor", "Morgan", "Chris", "Sam", "Riley", "Casey", "Dakota", "Jamie","Avery", "Reese", "Skyler", "Quinn", "Rowan", "Peyton", "Finley", "Emerson", "Hayden", "Sage","Logan", "Jesse", "Harper", "Eden", "Kendall", "Devon", "Dallas", "Shiloh", "River", "Phoenix",- "Cameron", "Drew", "Eli", "Francis", "Greyson", "Hadley", "Jules", "Kai", "Lennon", "Marlowe"+ "Cameron", "Drew", "Eli", "Francis", "Greyson", "Hadley", "Jules", "Kai", "Lennon", "Marlowe",];+// prettier-ignoreconst LAST_NAMES = ["Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Rodriguez", "Martinez","Hernandez", "Lopez", "Gonzalez", "Wilson", "Anderson", "Thomas", "Taylor", "Moore", "Jackson", "Martin","Lee", "Perez", "Thompson", "White", "Harris", "Sanchez", "Clark", "Ramirez", "Lewis", "Robinson",- "Walker", "Young", "Allen", "King", "Wright", "Scott", "Torres", "Nguyen", "Hill", "Flores"+ "Walker", "Young", "Allen", "King", "Wright", "Scott", "Torres", "Nguyen", "Hill", "Flores",];+// prettier-ignoreconst COMPANY_SUFFIXES = ["Tech", "Labs", "Media", "Agency", "Studio", "Ventures", "Digital", "Creative", "Global", "Solutions",- "Interactive", "Capital", "Partners", "Holdings", "Group", "Network", "Cloud", "AI", "Analytics", "Growth"+ "Interactive", "Capital", "Partners", "Holdings", "Group", "Network", "Cloud", "AI", "Analytics", "Growth",];+// prettier-ignoreconst DOMAINS = ["example.com", "techcorp.io", "marketing.co", "acme.dev", "growth.app",- "agency.net", "saas.com", "creator.xyz", "dub.co", "builder.build"+ "agency.net", "saas.com", "creator.xyz", "dub.co", "builder.build",];+// prettier-ignoreconst COUNTRIES = ["US", "CA", "GB", "DE", "FR", "AU", "JP", "IN", "BR", "NL", "ES", "SE", "SG"];const DESCRIPTIONS = [@@ -39,7 +72,7 @@ const DESCRIPTIONS = ["Social media influencer creating tech reviews, tutorials, and unboxings.","E-commerce consultant helping brands scale through affiliate networks.","Newsletter creator focused on modern web development and AI tools.",- "Community leader running a developer network and podcast."+ "Community leader running a developer network and podcast.",];const PLATFORM_TYPES: PlatformType[] = [@@ -48,34 +81,291 @@ const PLATFORM_TYPES: PlatformType[] = [PlatformType.twitter,PlatformType.linkedin,PlatformType.instagram,- PlatformType.tiktok+ PlatformType.tiktok,];-async function main() {- const args = process.argv.slice(2);- let totalCount = 100000;+type SeedArguments = {+ totalCount: number;+ targetProgramId: string | null;+ seed: string;+};++type PartnerChunk = {+ users: Prisma.UserCreateManyInput[];+ partners: Prisma.PartnerCreateManyInput[];+ partnerUsers: Prisma.PartnerUserCreateManyInput[];+ enrollments: Prisma.ProgramEnrollmentCreateManyInput[];+ platforms: Prisma.PartnerPlatformCreateManyInput[];+ links: Prisma.LinkCreateManyInput[];+};++type GeneratePartnerChunkOptions = {+ start: number;+ end: number;+ seedNamespace: string;+ seedFingerprint: string;+ passwordHash: string;+ runStartedAt: Date;+ programId: string;+ defaultGroupId: string | null;+ programDomain: string | null;+ workspaceId: string;+};++// These helpers are intentionally file-local; this script is their only caller.+const parseArguments = (args: string[]): SeedArguments => {+ let totalCount = DEFAULT_COUNT;let targetProgramId: string | null = null;+ let seed = DEFAULT_SEED;for (const arg of args) {if (arg.startsWith("--count=")) {- totalCount = parseInt(arg.split("=")[1], 10) || 100000;+ totalCount = parsePositiveInteger(arg.split("=")[1], "--count");} else if (arg.startsWith("--programId=")) {targetProgramId = arg.split("=")[1];+ } else if (arg.startsWith("--seed=")) {+ seed = arg.split("=")[1];+ } else {+ throw new Error(`Unknown argument: ${arg}`);+ }+ }++ if (!seed || !/^[a-zA-Z0-9_-]{1,40}$/.test(seed)) {+ throw new Error(+ "--seed must contain 1-40 letters, numbers, underscores, or hyphens.",+ );+ }++ if (targetProgramId === "") {+ throw new Error("--programId cannot be empty.");+ }++ return { totalCount, targetProgramId, seed };+};++const resolveProgramId = async (targetProgramId: string | null) => {+ if (targetProgramId) {+ return targetProgramId;+ }++ const programs = await prisma.program.findMany({+ take: 2,+ select: { id: true, name: true },+ });++ if (programs.length > 1) {+ throw new Error(+ `Multiple programs found. Pass --programId=<id> to choose one (for example, ${programs[0].id} for "${programs[0].name}").`,+ );+ }++ return programs[0]?.id ?? null;+};++const generatePartnerChunk = ({+ start,+ end,+ seedNamespace,+ seedFingerprint,+ passwordHash,+ runStartedAt,+ programId,+ defaultGroupId,+ programDomain,+ workspaceId,+}: GeneratePartnerChunkOptions): PartnerChunk => {+ const users: Prisma.UserCreateManyInput[] = [];+ const partners: Prisma.PartnerCreateManyInput[] = [];+ const partnerUsers: Prisma.PartnerUserCreateManyInput[] = [];+ const enrollments: Prisma.ProgramEnrollmentCreateManyInput[] = [];+ const platforms: Prisma.PartnerPlatformCreateManyInput[] = [];+ const links: Prisma.LinkCreateManyInput[] = [];++ for (let i = start; i < end; i++) {+ const partnerId = createDeterministicId({+ namespace: seedNamespace,+ prefix: "pn_",+ index: i,+ });+ const userId = createDeterministicId({+ namespace: seedNamespace,+ prefix: "user_",+ index: i,+ });+ const enrollmentId = createDeterministicId({+ namespace: seedNamespace,+ prefix: "pge_",+ index: i,+ });++ const firstName = FIRST_NAMES[i % FIRST_NAMES.length];+ const lastName =+ LAST_NAMES[(i + Math.floor(i / FIRST_NAMES.length)) % LAST_NAMES.length];+ const name = `${firstName} ${lastName}`;++ // Include deliberate prefix and infix cases for search verification.+ let emailPrefix: string;+ if (i % 100 === 0) {+ emailPrefix = `partner.${seedFingerprint}.${i}`;+ } else if (i % 137 === 0) {+ emailPrefix = `substringneedle.${seedFingerprint}.${i}`;+ } else if (i % 75 === 0) {+ emailPrefix = `tech.creator.${seedFingerprint}.${i}`;+ } else if (i % 50 === 0) {+ emailPrefix = `dub.affiliate.${seedFingerprint}.${i}`;+ } else {+ emailPrefix = `${firstName.toLowerCase()}.${lastName.toLowerCase()}.${seedFingerprint}.${i}`;+ }++ const domain = DOMAINS[i % DOMAINS.length];+ const email = `${emailPrefix}@${domain}`;+ const username = `${firstName.toLowerCase()}_${lastName.toLowerCase()}_${seedFingerprint}_${i}`;+ const companyName = `${lastName} ${COMPANY_SUFFIXES[i % COMPANY_SUFFIXES.length]}`;+ const country = COUNTRIES[i % COUNTRIES.length];+ const description = DESCRIPTIONS[i % DESCRIPTIONS.length];+ const createdAt = new Date(+ runStartedAt.getTime() - ((i * 60_000) % (365 * 86_400_000)),+ );++ users.push({+ id: userId,+ name,+ email,+ emailVerified: runStartedAt,+ passwordHash,+ defaultPartnerId: partnerId,+ createdAt,+ });++ partners.push({+ id: partnerId,+ name,+ username,+ email,+ description,+ country,+ companyName,+ networkStatus: "approved",+ createdAt,+ });++ partnerUsers.push({+ id: createDeterministicId({+ namespace: seedNamespace,+ prefix: "pnusr_",+ index: i,+ }),+ userId,+ partnerId,+ role: "owner",+ createdAt,+ });++ enrollments.push({+ id: enrollmentId,+ partnerId,+ programId,+ groupId: defaultGroupId,+ status: "approved",+ createdAt,+ });++ // Give every partner one or two records across all supported platforms.+ const numPlatforms = 1 + (i % 2);+ for (let p = 0; p < numPlatforms; p++) {+ const platformType = PLATFORM_TYPES[(i + p) % PLATFORM_TYPES.length];+ const identifier =+ platformType === PlatformType.website+ ? `https://www.${companyName.toLowerCase().replace(/[^a-z0-9]/g, "")}-${i}.${domain}`+ : `@${firstName.toLowerCase()}_${lastName.toLowerCase()}_${i}`;++ platforms.push({+ id: createDeterministicId({+ namespace: seedNamespace,+ prefix: `pnp${p}_`,+ index: i,+ }),+ partnerId,+ type: platformType,+ identifier,+ subscribers: BigInt(100 + ((i * 37) % 50000)),+ views: BigInt(500 + ((i * 123) % 500000)),+ verifiedAt: createdAt,+ createdAt,+ });}++ // Give every partner one searchable referral link.+ const linkKey = `p-${seedFingerprint}-${i}`;+ const linkDomain = programDomain || "dub.sh";+ links.push({+ id: createDeterministicId({+ namespace: seedNamespace,+ prefix: "link_",+ index: i,+ }),+ domain: linkDomain,+ key: linkKey,+ url: `https://${domain}/ref/${username}`,+ shortLink: `https://${linkDomain}/${linkKey}`,+ projectId: workspaceId,+ programId,+ partnerId,+ createdAt,+ });}- console.log(`\nπ Starting 100K+ Partner Data Seed (Target Count: ${totalCount.toLocaleString()})...`);+ return { users, partners, partnerUsers, enrollments, platforms, links };+};++const insertPartnerChunk = async ({+ users,+ partners,+ partnerUsers,+ enrollments,+ platforms,+ links,+}: PartnerChunk) => {+ // Keep every chunk atomic. Stable IDs make the same seed safe to resume/rerun.+ const [, partnerResult] = await prisma.$transaction([+ prisma.user.createMany({ data: users, skipDuplicates: true }),+ prisma.partner.createMany({ data: partners, skipDuplicates: true }),+ prisma.partnerUser.createMany({ data: partnerUsers, skipDuplicates: true }),+ prisma.programEnrollment.createMany({+ data: enrollments,+ skipDuplicates: true,+ }),+ prisma.partnerPlatform.createMany({+ data: platforms,+ skipDuplicates: true,+ }),+ prisma.link.createMany({ data: links, skipDuplicates: true }),+ ]);++ return partnerResult.count;+};++async function main() {+ // Step 1: Parse and validate command-line options.+ const { totalCount, targetProgramId, seed } = parseArguments(+ process.argv.slice(2),+ );++ // Step 2: Resolve the target program and its workspace.+ const resolvedProgramId = await resolveProgramId(targetProgramId);- // Find target program & workspace- const program = targetProgramId- ? await prisma.program.findUnique({ where: { id: targetProgramId } })- : await prisma.program.findFirst();+ console.log(+ `\nπ Starting Partner Data Seed (Target Count: ${totalCount.toLocaleString()}, Seed: "${seed}")...`,+ );++ const program = resolvedProgramId+ ? await prisma.program.findUnique({ where: { id: resolvedProgramId } })+ : null;if (!program) {- console.error(- "β No program found in database. Please run 'pnpm run script dev/seed' first to set up the default workspace and program."+ throw new Error(+ "β No program found in database. Please run 'pnpm run script dev/seed' first to set up the default workspace and program.",);- process.exit(1);}const workspace = await prisma.project.findUnique({@@ -83,157 +373,58 @@ async function main() {});if (!workspace) {- console.error("β Program workspace not found.");- process.exit(1);+ throw new Error("β Program workspace not found.");}- console.log(`π Seeding partners for Program: "${program.name}" (${program.id})`);+ console.log(+ `π Seeding partners for Program: "${program.name}" (${program.id})`,+ );console.log(` Workspace: "${workspace.name}" (${workspace.id})\n`);+ // Step 3: Build the stable namespace shared by every generated chunk.const passwordHash = await hashPassword("password");- const chunkSize = 2500;- const totalChunks = Math.ceil(totalCount / chunkSize);+ const seedNamespace = `${program.id}:${seed}`;+ const seedFingerprint = createHash("sha256")+ .update(seedNamespace)+ .digest("hex")+ .slice(0, 10);+ const runStartedAt = new Date();+ const totalChunks = Math.ceil(totalCount / CHUNK_SIZE);const startTime = Date.now();+ let insertedPartners = 0;+ // Step 4: Generate and atomically insert one bounded chunk at a time.for (let chunk = 0; chunk < totalChunks; chunk++) {- const chunkStart = chunk * chunkSize;- const chunkEnd = Math.min(chunkStart + chunkSize, totalCount);- const currentChunkSize = chunkEnd - chunkStart;-- const chunkUsers: Prisma.UserCreateManyInput[] = [];- const chunkPartners: Prisma.PartnerCreateManyInput[] = [];- const chunkPartnerUsers: Prisma.PartnerUserCreateManyInput[] = [];- const chunkEnrollments: Prisma.ProgramEnrollmentCreateManyInput[] = [];- const chunkPlatforms: Prisma.PartnerPlatformCreateManyInput[] = [];- const chunkLinks: Prisma.LinkCreateManyInput[] = [];-- for (let i = chunkStart; i < chunkEnd; i++) {- const partnerId = createId({ prefix: "pn_" });- const userId = createId({ prefix: "user_" });- const enrollmentId = createId({ prefix: "pge_" });-- const firstName = FIRST_NAMES[i % FIRST_NAMES.length];- const lastName = LAST_NAMES[(i + Math.floor(i / FIRST_NAMES.length)) % LAST_NAMES.length];- const name = `${firstName} ${lastName}`;-- const runSuffix = Math.floor(Date.now() / 1000).toString(36);- // Email generation with deliberate test cases for search verification- let emailPrefix: string;- if (i % 100 === 0) {- emailPrefix = `examp.partner.${runSuffix}.${i}`; // Ensures partial match 'examp' works- } else if (i % 75 === 0) {- emailPrefix = `tech.creator.${runSuffix}.${i}`;- } else if (i % 50 === 0) {- emailPrefix = `dub.affiliate.${runSuffix}.${i}`;- } else {- emailPrefix = `${firstName.toLowerCase()}.${lastName.toLowerCase()}.${runSuffix}.${i}`;- }-- const domain = DOMAINS[i % DOMAINS.length];- const email = `${emailPrefix}@${domain}`;- const username = `${firstName.toLowerCase()}_${lastName.toLowerCase()}_${runSuffix}_${i}`;- const companyName = `${lastName} ${COMPANY_SUFFIXES[i % COMPANY_SUFFIXES.length]}`;- const country = COUNTRIES[i % COUNTRIES.length];- const description = DESCRIPTIONS[i % DESCRIPTIONS.length];- const createdAt = new Date(Date.now() - (i * 60000) % (365 * 86400000));-- // User record- chunkUsers.push({- id: userId,- name,- email,- emailVerified: new Date(),- passwordHash,- defaultPartnerId: partnerId,- createdAt,- });-- // Partner record- chunkPartners.push({- id: partnerId,- name,- username,- email,- description,- country,- companyName,- networkStatus: "approved",- createdAt,- });-- // PartnerUser record- chunkPartnerUsers.push({- id: createId({ prefix: "pn_" }), // or unique ID- userId,- partnerId,- role: "owner",- createdAt,- });-- // Program Enrollment record- chunkEnrollments.push({- id: enrollmentId,- partnerId,- programId: program.id,- groupId: program.defaultGroupId,- status: "approved",- createdAt,- });-- // Generate 1-2 Platforms per partner- const numPlatforms = 1 + (i % 2);- for (let p = 0; p < numPlatforms; p++) {- const platformType = PLATFORM_TYPES[(i + p) % PLATFORM_TYPES.length];- const identifier =- platformType === PlatformType.website- ? `https://www.${companyName.toLowerCase().replace(/[^a-z0-9]/g, "")}-${i}.${domain}`- : `@${firstName.toLowerCase()}_${lastName.toLowerCase()}_${i}`;-- chunkPlatforms.push({- id: createId({ prefix: "pn_" }),- partnerId,- type: platformType,- identifier,- subscribers: BigInt(100 + ((i * 37) % 50000)),- views: BigInt(500 + ((i * 123) % 500000)),- verifiedAt: createdAt,- createdAt,- });- }-- // Generate 1 Short Link per partner- const linkKey = `p-${firstName.toLowerCase()}-${lastName.toLowerCase()}-${i}`;- const linkDomain = program.domain || "dub.sh";- chunkLinks.push({- id: createId({ prefix: "link_" }),- domain: linkDomain,- key: linkKey,- url: `https://${domain}/ref/${username}`,- shortLink: `https://${linkDomain}/${linkKey}`,- projectId: workspace.id,- programId: program.id,- partnerId,- createdAt,- });- }-- // Perform bulk insertions- await prisma.user.createMany({ data: chunkUsers, skipDuplicates: true });- await prisma.partner.createMany({ data: chunkPartners, skipDuplicates: true });- await prisma.partnerUser.createMany({ data: chunkPartnerUsers, skipDuplicates: true });- await prisma.programEnrollment.createMany({ data: chunkEnrollments, skipDuplicates: true });- await prisma.partnerPlatform.createMany({ data: chunkPlatforms, skipDuplicates: true });- await prisma.link.createMany({ data: chunkLinks, skipDuplicates: true });-+ const chunkStart = chunk * CHUNK_SIZE;+ const chunkEnd = Math.min(chunkStart + CHUNK_SIZE, totalCount);+ const partnerChunk = generatePartnerChunk({+ start: chunkStart,+ end: chunkEnd,+ seedNamespace,+ seedFingerprint,+ passwordHash,+ runStartedAt,+ programId: program.id,+ defaultGroupId: program.defaultGroupId,+ programDomain: program.domain,+ workspaceId: workspace.id,+ });+ const insertedInChunk = await insertPartnerChunk(partnerChunk);+ insertedPartners += insertedInChunk;++ // Report both processing progress and the idempotent insert count.const elapsedSec = ((Date.now() - startTime) / 1000).toFixed(1);const progressPct = (((chunk + 1) / totalChunks) * 100).toFixed(0);console.log(- ` [Chunk ${chunk + 1}/${totalChunks}] (${progressPct}%) Inserted ${chunkEnd.toLocaleString()}/${totalCount.toLocaleString()} partners... (${elapsedSec}s elapsed)`+ ` [Chunk ${chunk + 1}/${totalChunks}] (${progressPct}%) Processed ${chunkEnd.toLocaleString()}/${totalCount.toLocaleString()} partners (${insertedInChunk.toLocaleString()} new)... (${elapsedSec}s elapsed)`,);}+ // Step 5: Summarize how much of the requested seed was newly inserted.const totalTimeSec = ((Date.now() - startTime) / 1000).toFixed(1);- console.log(`\nβ Successfully seeded ${totalCount.toLocaleString()} partners in ${totalTimeSec} seconds!`);+ console.log(+ `\nβ Seed complete: ${insertedPartners.toLocaleString()} new, ${(totalCount - insertedPartners).toLocaleString()} already present (${totalTimeSec}s).`,+ );}main()