The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
update scripts
Steven Teyef2d2cc0af0b
diff --git a/apps/web/lib/middleware/utils/get-final-url.ts b/apps/web/lib/middleware/utils/get-final-url.tsindex c39b3ddd08..e4a082a5cf 100644--- a/apps/web/lib/middleware/utils/get-final-url.ts+++ b/apps/web/lib/middleware/utils/get-final-url.ts@@ -38,7 +38,7 @@ export const getFinalUrl = (if (clickId) {/*- custom query param for stripe payment links + Dub Conversions+ custom query param for stripe payment links:- if there is a clickId and dub_client_reference_id is 1- then set client_reference_id to dub_id_${clickId} and drop the dub_client_reference_id param- our Stripe integration will then detect `dub_id_${clickId}` as the dubClickId in the `checkout.session.completed` webhookdiff --git a/apps/web/scripts/cal/backfill-referral-links.ts b/apps/web/scripts/cal/backfill-referral-links.tsdeleted file mode 100644index d603adb5c1..0000000000--- a/apps/web/scripts/cal/backfill-referral-links.ts+++ /dev/null@@ -1,142 +0,0 @@-import { PlanProps } from "@/lib/types";-import { prisma } from "@dub/prisma";-import "dotenv-flow/config";-import * as fs from "fs";-import * as Papa from "papaparse";-import { includeTags } from "../../lib/api/links/include-tags";-import { createAndEnrollPartner } from "../../lib/api/partners/create-and-enroll-partner";-import { recordLink } from "../../lib/tinybird/record-link";--interface BackfillLinkProp {- externalId: string;- key: string;- name: string;- email: string;- avatar?: string;-}--const linksToBackfill: BackfillLinkProp[] = [];--async function main() {- const { workspace, ...program } = await prisma.program.findUniqueOrThrow({- where: {- id: "prog_mODHMDrJPWlkpT7uzsUASFhK",- },- include: {- workspace: true,- },- });-- Papa.parse(fs.createReadStream("refer_cal_links_backfilled.csv", "utf-8"), {- header: true,- skipEmptyLines: true,- step: (result: { data: BackfillLinkProp }) => {- linksToBackfill.push(result.data);- },- complete: async () => {- const batch = linksToBackfill.slice(0, 50);-- const finalResults: {- name: string;- email: string;- clicks: number;- leads: number;- sales: number;- }[] = [];-- await Promise.all(- batch.map(async (l) => {- const link = await prisma.link.findUnique({- where: {- domain_key: {- domain: "refer.cal.com",- key: l.key,- },- },- });-- if (!link) {- console.log("Link not found", l.email);- return;- }-- if (link.partnerId) {- console.log("Partner already enrolled", l.email);- return;- }-- try {- const res = await createAndEnrollPartner({- workspace: {- ...workspace,- plan: workspace.plan as PlanProps,- webhookEnabled: false,- },- program,- partner: {- name: l.name,- email: l.email,- image: l.avatar && l.avatar.length < 190 ? l.avatar : undefined,- tenantId: l.externalId,- },- // @ts-ignore- link,- userId: "",- skipEnrollmentCheck: true,- });-- finalResults.push({- name: res.name,- email: res.email ?? "",- clicks: res.totalClicks,- leads: res.totalLeads,- sales: res.totalSales,- });-- // remove row from csv- linksToBackfill.splice(linksToBackfill.indexOf(l), 1);- fs.writeFileSync(- "refer_cal_links_backfilled.csv",- Papa.unparse(linksToBackfill),- "utf-8",- );- } catch (e) {- if (e.message.includes("already enrolled")) {- const partner = await prisma.partner.update({- where: {- email: l.email,- },- data: {- name: l.name,- image: l.avatar ?? undefined,- },- });-- await prisma.link- .update({- where: {- id: link.id,- },- data: {- programId: program.id,- partnerId: partner.id,- tenantId: l.externalId,- folderId: program.defaultFolderId,- trackConversion: true,- },- include: includeTags,- })- .then((link) => recordLink(link));- } else {- console.error(e, l.email);- }- }- }),- );-- console.table(finalResults);- },- });-}--main();diff --git a/apps/web/scripts/annature/import-domains.ts b/apps/web/scripts/customers/annature/import-domains.tssimilarity index 100%rename from apps/web/scripts/annature/import-domains.tsrename to apps/web/scripts/customers/annature/import-domains.tsdiff --git a/apps/web/scripts/buffer/delete-old-links.ts b/apps/web/scripts/customers/buffer/delete-old-links.tssimilarity index 100%rename from apps/web/scripts/buffer/delete-old-links.tsrename to apps/web/scripts/customers/buffer/delete-old-links.tsdiff --git a/apps/web/scripts/buffer/migrate-to-case-sensitive.ts b/apps/web/scripts/customers/buffer/migrate-to-case-sensitive.tssimilarity index 94%rename from apps/web/scripts/buffer/migrate-to-case-sensitive.tsrename to apps/web/scripts/customers/buffer/migrate-to-case-sensitive.tsindex 778b8ee98d..62264f0ab8 100644--- a/apps/web/scripts/buffer/migrate-to-case-sensitive.ts+++ b/apps/web/scripts/customers/buffer/migrate-to-case-sensitive.ts@@ -2,8 +2,8 @@ import { prisma } from "@dub/prisma";import { Prisma } from "@dub/prisma/client";import { linkConstructorSimple } from "@dub/utils";import "dotenv-flow/config";-import { linkCache } from "../../lib/api/links/cache";-import { encodeKeyIfCaseSensitive } from "../../lib/api/links/case-sensitivity";+import { linkCache } from "../../../lib/api/links/cache";+import { encodeKeyIfCaseSensitive } from "../../../lib/api/links/case-sensitivity";const domain = "buff.ly";const userId = "user_EzRuKzR9sG3WmHapVV6aEec7";diff --git a/apps/web/scripts/framer/1-process-framer-combined.ts b/apps/web/scripts/customers/framer/1-process-framer-combined.tssimilarity index 100%rename from apps/web/scripts/framer/1-process-framer-combined.tsrename to apps/web/scripts/customers/framer/1-process-framer-combined.tsdiff --git a/apps/web/scripts/framer/2-sort-lead-events-by-date.ts b/apps/web/scripts/customers/framer/2-sort-lead-events-by-date.tssimilarity index 100%rename from apps/web/scripts/framer/2-sort-lead-events-by-date.tsrename to apps/web/scripts/customers/framer/2-sort-lead-events-by-date.tsdiff --git a/apps/web/scripts/framer/3-backfill-tb-events.ts b/apps/web/scripts/customers/framer/3-backfill-tb-events.tssimilarity index 100%rename from apps/web/scripts/framer/3-backfill-tb-events.tsrename to apps/web/scripts/customers/framer/3-backfill-tb-events.tsdiff --git a/apps/web/scripts/framer/backfill-commissions.ts b/apps/web/scripts/customers/framer/backfill-commissions.tssimilarity index 100%rename from apps/web/scripts/framer/backfill-commissions.tsrename to apps/web/scripts/customers/framer/backfill-commissions.tsdiff --git a/apps/web/scripts/framer/check-pending-payout-totals.ts b/apps/web/scripts/customers/framer/check-pending-payout-totals.tssimilarity index 100%rename from apps/web/scripts/framer/check-pending-payout-totals.tsrename to apps/web/scripts/customers/framer/check-pending-payout-totals.tsdiff --git a/apps/web/scripts/framer/get-links-to-backfill.ts b/apps/web/scripts/customers/framer/get-links-to-backfill.tssimilarity index 100%rename from apps/web/scripts/framer/get-links-to-backfill.tsrename to apps/web/scripts/customers/framer/get-links-to-backfill.tsdiff --git a/apps/web/scripts/framer/get-remaining-links-to-backfill.ts b/apps/web/scripts/customers/framer/get-remaining-links-to-backfill.tssimilarity index 100%rename from apps/web/scripts/framer/get-remaining-links-to-backfill.tsrename to apps/web/scripts/customers/framer/get-remaining-links-to-backfill.tsdiff --git a/apps/web/scripts/framer/mark-commissions-paid.ts b/apps/web/scripts/customers/framer/mark-commissions-paid.tssimilarity index 100%rename from apps/web/scripts/framer/mark-commissions-paid.tsrename to apps/web/scripts/customers/framer/mark-commissions-paid.tsdiff --git a/apps/web/scripts/framer/mark-commissions-pending.ts b/apps/web/scripts/customers/framer/mark-commissions-pending.tssimilarity index 100%rename from apps/web/scripts/framer/mark-commissions-pending.tsrename to apps/web/scripts/customers/framer/mark-commissions-pending.tsdiff --git a/apps/web/scripts/framer/process-lead-events.ts b/apps/web/scripts/customers/framer/process-lead-events.tssimilarity index 100%rename from apps/web/scripts/framer/process-lead-events.tsrename to apps/web/scripts/customers/framer/process-lead-events.tsdiff --git a/apps/web/scripts/framer/tally-commissions.ts b/apps/web/scripts/customers/framer/tally-commissions.tssimilarity index 100%rename from apps/web/scripts/framer/tally-commissions.tsrename to apps/web/scripts/customers/framer/tally-commissions.tsdiff --git a/apps/web/scripts/perplexity/backfill-leads.ts b/apps/web/scripts/customers/perplexity/backfill-leads.tssimilarity index 100%rename from apps/web/scripts/perplexity/backfill-leads.tsrename to apps/web/scripts/customers/perplexity/backfill-leads.tsdiff --git a/apps/web/scripts/perplexity/backfill-tenantids.ts b/apps/web/scripts/customers/perplexity/backfill-tenantids.tssimilarity index 100%rename from apps/web/scripts/perplexity/backfill-tenantids.tsrename to apps/web/scripts/customers/perplexity/backfill-tenantids.tsdiff --git a/apps/web/scripts/perplexity/ban-partners.ts b/apps/web/scripts/customers/perplexity/ban-partners.tssimilarity index 100%rename from apps/web/scripts/perplexity/ban-partners.tsrename to apps/web/scripts/customers/perplexity/ban-partners.tsdiff --git a/apps/web/scripts/perplexity/deactivate-partners.ts b/apps/web/scripts/customers/perplexity/deactivate-partners.tssimilarity index 100%rename from apps/web/scripts/perplexity/deactivate-partners.tsrename to apps/web/scripts/customers/perplexity/deactivate-partners.tsdiff --git a/apps/web/scripts/perplexity/move-partners.ts b/apps/web/scripts/customers/perplexity/move-partners.tssimilarity index 100%rename from apps/web/scripts/perplexity/move-partners.tsrename to apps/web/scripts/customers/perplexity/move-partners.tsdiff --git a/apps/web/scripts/perplexity/partners-updated-countries.ts b/apps/web/scripts/customers/perplexity/partners-updated-countries.tssimilarity index 100%rename from apps/web/scripts/perplexity/partners-updated-countries.tsrename to apps/web/scripts/customers/perplexity/partners-updated-countries.tsdiff --git a/apps/web/scripts/perplexity/review-bounties.ts b/apps/web/scripts/customers/perplexity/review-bounties.tssimilarity index 100%rename from apps/web/scripts/perplexity/review-bounties.tsrename to apps/web/scripts/customers/perplexity/review-bounties.tsdiff --git a/apps/web/scripts/perplexity/update-commissions.ts b/apps/web/scripts/customers/perplexity/update-commissions.tssimilarity index 100%rename from apps/web/scripts/perplexity/update-commissions.tsrename to apps/web/scripts/customers/perplexity/update-commissions.tsdiff --git a/apps/web/scripts/perplexity/update-notifications.ts b/apps/web/scripts/customers/perplexity/update-notifications.tssimilarity index 100%rename from apps/web/scripts/perplexity/update-notifications.tsrename to apps/web/scripts/customers/perplexity/update-notifications.tsdiff --git a/apps/web/scripts/customers/scaledmail/export-invoicecs-by-stripe-id.ts b/apps/web/scripts/customers/scaledmail/export-invoicecs-by-stripe-id.tsnew file mode 100644index 0000000000..e109fc411b--- /dev/null+++ b/apps/web/scripts/customers/scaledmail/export-invoicecs-by-stripe-id.ts@@ -0,0 +1,63 @@+import { prisma } from "@dub/prisma";+import "dotenv-flow/config";+import * as fs from "fs";+import * as Papa from "papaparse";+import { stripeAppClient } from "../../../lib/stripe";++const programId = "prog_xxx";+const customerIdsToImport: string[] = [];++// script to export stripe invoices based on the customer's stripeCustomerId+async function main() {+ console.log(`Found ${customerIdsToImport.length} paying customers`);+ const program = await prisma.program.findUniqueOrThrow({+ where: {+ id: programId,+ },+ include: {+ workspace: true,+ },+ });++ console.log(`Found ${customerIdsToImport.length} paying customers`);++ for (const customerId of customerIdsToImport) {+ const invoices = await stripeAppClient({+ mode: "live",+ }).invoices.list(+ {+ customer: customerId,+ status: "paid",+ },+ {+ stripeAccount: program.workspace.stripeConnectId!,+ },+ );++ const invoicesToBackfill = invoices.data.map((invoice) => ({+ customerExternalId: customerId,+ invoiceId: invoice.id,+ amountPaid: invoice.amount_paid,+ createdAt: new Date(invoice.created * 1000).toISOString(),+ }));++ if (invoicesToBackfill.length > 0) {+ console.table(invoicesToBackfill, [+ "customerExternalId",+ "invoiceId",+ "amountPaid",+ "createdAt",+ ]);++ const filePath = "scaledmail_customer_stripe_invoices.csv";+ const fileExists = fs.existsSync(filePath);++ fs.appendFileSync(+ filePath,+ Papa.unparse(invoicesToBackfill, { header: !fileExists }) + "\n",+ );+ }+ }+}++main();diff --git a/apps/web/scripts/customers/scaledmail/import-customers-by-stripe-id.ts b/apps/web/scripts/customers/scaledmail/import-customers-by-stripe-id.tsnew file mode 100644index 0000000000..f9b5f1c943--- /dev/null+++ b/apps/web/scripts/customers/scaledmail/import-customers-by-stripe-id.ts@@ -0,0 +1,279 @@+import { createId } from "@/lib/api/create-id";+import { prisma } from "@dub/prisma";+import { Prisma } from "@dub/prisma/client";+import { chunk, nanoid, prettyPrint } from "@dub/utils";+import "dotenv-flow/config";+import { syncPartnerLinksStats } from "../../../lib/api/partners/sync-partner-links-stats";+import { stripeAppClient } from "../../../lib/stripe";+import { recordLeadWithTimestamp } from "../../../lib/tinybird/record-lead";++const programId = "prog_xxx";+const linkId = "link_xxx";+const customerIdsToImport: string[] = [];++// script to backfill customers + leads+// we also use a batching logic for tinybird events ingestion+async function main() {+ const program = await prisma.program.findUniqueOrThrow({+ where: {+ id: programId,+ },+ include: {+ workspace: true,+ },+ });+ const { workspace } = program;++ const link = await prisma.link.findUniqueOrThrow({+ where: {+ id: linkId,+ },+ });++ const existingCustomers = await prisma.customer.findMany({+ where: {+ stripeCustomerId: {+ in: customerIdsToImport,+ },+ },+ });++ const finalCustomerIdsToImport = customerIdsToImport.filter(+ (customerId) =>+ !existingCustomers.some(+ (customer) => customer.stripeCustomerId === customerId,+ ),+ );++ let finalLeadsToBackfill: {+ customerExternalId: string;+ stripeCustomerId: string;+ name: string | null;+ email: string | null;+ country: string;+ timestamp: Date;+ }[] = [];++ for (const customerId of finalCustomerIdsToImport) {+ const customer = await stripeAppClient({+ mode: "live",+ }).customers.retrieve(customerId, {+ stripeAccount: program.workspace.stripeConnectId!,+ });++ console.log(customer);+ if (customer.deleted) {+ continue;+ }++ finalLeadsToBackfill.push({+ customerExternalId: customer.id,+ stripeCustomerId: customerId,+ name: customer.name ?? null,+ email: customer.email,+ country: customer.address?.country ?? "US",+ timestamp: new Date(customer.created * 1000),+ });+ }++ console.log(`Found ${finalLeadsToBackfill.length} leads to backfill`);+ console.table(finalLeadsToBackfill.slice(0, 10));++ const clicksToCreate = finalLeadsToBackfill+ .map((lead) => {+ const clickId = nanoid(16);++ return {+ timestamp: new Date(lead.timestamp).toISOString(),+ identity_hash: lead.customerExternalId,+ click_id: clickId,+ workspace_id: workspace.id,+ link_id: link.id,+ domain: link.domain,+ key: link.key,+ url: link.url,+ ip: "",+ continent: "NA",+ country: lead.country,+ region: "Unknown",+ city: "Unknown",+ latitude: "Unknown",+ longitude: "Unknown",+ vercel_region: "",+ device: "Desktop",+ device_vendor: "Unknown",+ device_model: "Unknown",+ browser: "Unknown",+ browser_version: "Unknown",+ engine: "Unknown",+ engine_version: "Unknown",+ os: "Unknown",+ os_version: "Unknown",+ cpu_architecture: "Unknown",+ ua: "Unknown",+ bot: 0,+ qr: 0,+ referer: "(direct)",+ referer_url: "(direct)",+ trigger: "link",+ };+ })+ .filter((p): p is NonNullable<typeof p> => p !== null);++ // clickhouse only supports max 12 partitions (months) for a given event backfill+ // so we need to transform this into a list of lists, one for each year+ const clicksToCreateTB = clicksToCreate.reduce(+ (acc, curr) => {+ const year = new Date(curr.timestamp).getFullYear();+ if (!acc[year]) {+ acc[year] = [];+ }+ acc[year].push(curr);+ return acc;+ },+ {} as Record<number, any[]>,+ );++ // Record clicks+ Object.entries(clicksToCreateTB).forEach(async ([year, clicks]) => {+ const clicksBatch = clicks as typeof clicksToCreate;+ console.log(`backfilling ${clicksBatch.length} clicks for ${year}`);+ const clickRes = await fetch(+ `${process.env.TINYBIRD_API_URL}/v0/events?name=dub_click_events&wait=true`,+ {+ method: "POST",+ headers: {+ Authorization: `Bearer ${process.env.TINYBIRD_API_KEY}`,+ "Content-Type": "application/x-ndjson",+ },+ body: (clicksBatch as typeof clicksToCreate)+ .map((d) => JSON.stringify(d))+ .join("\n"),+ },+ ).then((res) => res.json());+ console.log("backfilled clicks", JSON.stringify(clickRes, null, 2));+ });++ const customersToCreate = finalLeadsToBackfill+ .map((lead, idx) => {+ const clickData = clicksToCreate[idx];+ if (!clickData) {+ return null;+ }+ return {+ id: createId({ prefix: "cus_" }),+ name: lead.name,+ email: lead.email,+ externalId: lead.customerExternalId,+ projectId: workspace.id,+ projectConnectId: workspace.stripeConnectId,+ stripeCustomerId: lead.stripeCustomerId,+ linkId: link.id,+ programId: link.programId,+ partnerId: link.partnerId,+ country: clickData.country,+ clickId: clickData.click_id,+ clickedAt: new Date(lead.timestamp).toISOString(),+ createdAt: new Date(lead.timestamp).toISOString(),+ };+ })+ .filter(+ (p): p is NonNullable<typeof p> => p !== null,+ ) satisfies Prisma.CustomerCreateManyInput[];++ console.table(customersToCreate.slice(0, 10));++ const customerRes = await prisma.customer.createMany({+ data: customersToCreate,+ skipDuplicates: true,+ });+ console.log("backfilled customers", prettyPrint(customerRes));++ const leadsToCreate = clicksToCreate.map((clickData, idx) => ({+ ...clickData,+ event_id: nanoid(16),+ event_name: "Sign up",+ customer_id: customersToCreate[idx]!.id,+ }));++ // same batching logic as above+ const leadsToCreateTB = leadsToCreate.reduce(+ (acc, curr) => {+ const year = new Date(curr.timestamp).getFullYear();+ if (!acc[year]) {+ acc[year] = [];+ }+ acc[year].push(curr);+ return acc;+ },+ {} as Record<number, any[]>,+ );++ Object.entries(leadsToCreateTB).forEach(async ([year, leads]) => {+ const leadsBatch = leads as typeof leadsToCreate;+ console.log(`backfilling ${leadsBatch.length} leads for ${year}`);+ const leadRes = await recordLeadWithTimestamp(leadsBatch);+ console.log("backfilled leads", prettyPrint(leadRes));+ });++ const statsByLink = finalLeadsToBackfill.reduce(+ (acc, lead) => {+ const leadCreatedAt = new Date(lead.timestamp);+ acc[link.id] = {+ clicks: (acc[link.id]?.clicks || 0) + 1,+ leads: (acc[link.id]?.leads || 0) + 1,+ // if there is no lastLeadAt, or the leadCreatedAt is greater than the lastLeadAt, set the lastLeadAt to the leadCreatedAt+ lastLeadAt:+ leadCreatedAt > (acc[link.id]?.lastLeadAt ?? new Date(0))+ ? leadCreatedAt+ : acc[link.id]?.lastLeadAt,+ };+ return acc;+ },+ {} as Record<+ string,+ {+ clicks: number;+ leads: number;+ lastLeadAt: Date | undefined;+ }+ >,+ );++ console.log(prettyPrint(Object.entries(statsByLink).slice(0, 10)));++ const statsByLinkChunks = chunk(Object.entries(statsByLink), 50);+ for (let i = 0; i < statsByLinkChunks.length; i++) {+ const chunk = statsByLinkChunks[i];+ console.log(+ `backfilling stats for ${chunk.length} links in batch ${i + 1} of ${statsByLinkChunks.length}`,+ );+ await Promise.all(+ chunk.map(async ([linkId, stats]) => {+ const res = await prisma.link.update({+ where: { id: linkId },+ data: {+ clicks: {+ increment: stats.clicks,+ },+ leads: {+ increment: stats.leads,+ },+ lastLeadAt: stats.lastLeadAt,+ },+ });+ console.log(+ `Updated ${linkId} to ${res.clicks} clicks (+${stats.clicks} clicks), ${res.leads} leads (+${stats.leads} leads)`,+ );+ const syncRes = await syncPartnerLinksStats({+ partnerId: res.partnerId!,+ programId: program.id,+ eventType: "lead",+ });+ console.log("synced stats", prettyPrint(syncRes));+ }),+ );+ }+}++main();diff --git a/apps/web/scripts/tella/remind-applications.ts b/apps/web/scripts/customers/tella/remind-applications.tssimilarity index 100%rename from apps/web/scripts/tella/remind-applications.tsrename to apps/web/scripts/customers/tella/remind-applications.tsdiff --git a/apps/web/scripts/tella/update-commission-flat.ts b/apps/web/scripts/customers/tella/update-commission-flat.tssimilarity index 100%rename from apps/web/scripts/tella/update-commission-flat.tsrename to apps/web/scripts/customers/tella/update-commission-flat.tsdiff --git a/apps/web/scripts/tella/update-commission-percentage.ts b/apps/web/scripts/customers/tella/update-commission-percentage.tssimilarity index 100%rename from apps/web/scripts/tella/update-commission-percentage.tsrename to apps/web/scripts/customers/tella/update-commission-percentage.tsdiff --git a/apps/web/scripts/tella/update-commissions.ts b/apps/web/scripts/customers/tella/update-commissions.tssimilarity index 100%rename from apps/web/scripts/tella/update-commissions.tsrename to apps/web/scripts/customers/tella/update-commissions.tsdiff --git a/apps/web/scripts/tella/update-reward-tier.ts b/apps/web/scripts/customers/tella/update-reward-tier.tssimilarity index 100%rename from apps/web/scripts/tella/update-reward-tier.tsrename to apps/web/scripts/customers/tella/update-reward-tier.tsdiff --git a/apps/web/scripts/wispr-flow/update-links.ts b/apps/web/scripts/customers/wispr-flow/update-links.tssimilarity index 91%rename from apps/web/scripts/wispr-flow/update-links.tsrename to apps/web/scripts/customers/wispr-flow/update-links.tsindex c7fe21e762..2faeccc9d4 100644--- a/apps/web/scripts/wispr-flow/update-links.ts+++ b/apps/web/scripts/customers/wispr-flow/update-links.ts@@ -1,6 +1,6 @@import { prisma } from "@dub/prisma";import "dotenv-flow/config";-import { linkCache } from "../../lib/api/links/cache";+import { linkCache } from "../../../lib/api/links/cache";// update linksasync function main() {diff --git a/apps/web/scripts/ship30/backfill-leads.ts b/apps/web/scripts/ship30/backfill-leads.tsdeleted file mode 100644index 0ed054aeee..0000000000--- a/apps/web/scripts/ship30/backfill-leads.ts+++ /dev/null@@ -1,260 +0,0 @@-// @ts-nocheck some weird typing issues below--import { createId } from "@/lib/api/create-id";-import { prisma } from "@dub/prisma";-import { nanoid } from "@dub/utils";-import "dotenv-flow/config";-import * as fs from "fs";-import * as Papa from "papaparse";-import { recordLeadWithTimestamp } from "../../lib/tinybird/record-lead";--let leadsToBackfill: {- customerExternalId: string;- partnerLinkKey: string;- timestamp: string;-}[] = [];--// script to backfill customers + leads-// we also use a batching logic for tinybird events ingestion-async function main() {- Papa.parse(fs.createReadStream("ship30_leads.csv", "utf-8"), {- header: true,- skipEmptyLines: true,- step: (result: {- data: {- customerExternalId: string;- partnerLinkKey: string;- timestamp: string;- };- }) => {- leadsToBackfill.push({- customerExternalId: result.data.customerExternalId,- partnerLinkKey: result.data.partnerLinkKey,- timestamp: new Date(result.data.timestamp).toISOString(),- });- },- complete: async () => {- const workspace = await prisma.project.findUniqueOrThrow({- where: {- id: "ws_xxx",- },- });-- const partnerLinks = await prisma.link.findMany({- where: {- domain: "ship30.partners",- key: {- in: leadsToBackfill.map((lead) => lead.partnerLinkKey),- },- },- });-- // filter out leads that are not associated with a partner link- const finalLeadsToBackfill = leadsToBackfill.filter((lead) =>- partnerLinks.some(- (link) =>- link.key.toLowerCase() === lead.partnerLinkKey.toLowerCase(),- ),- );-- console.log(`Found ${finalLeadsToBackfill.length} leads to backfill`);- console.table(finalLeadsToBackfill.slice(0, 10));-- const clicksToCreate = finalLeadsToBackfill- .map((lead) => {- const link = partnerLinks.find(- (link) =>- link.key.toLowerCase() === lead.partnerLinkKey.toLowerCase(),- )!; // coerce here cause we already filtered out leads that are not associated with a partner link above-- const clickId = nanoid(16);-- return {- timestamp: new Date(lead.timestamp).toISOString(),- identity_hash: lead.customerExternalId,- click_id: clickId,- workspace_id: workspace.id,- link_id: link.id,- domain: link.domain,- key: link.key,- url: link.url,- ip: "",- continent: "NA",- country: "US",- region: "Unknown",- city: "Unknown",- latitude: "Unknown",- longitude: "Unknown",- vercel_region: "",- device: "Desktop",- device_vendor: "Unknown",- device_model: "Unknown",- browser: "Unknown",- browser_version: "Unknown",- engine: "Unknown",- engine_version: "Unknown",- os: "Unknown",- os_version: "Unknown",- cpu_architecture: "Unknown",- ua: "Unknown",- bot: 0,- qr: 0,- referer: "(direct)",- referer_url: "(direct)",- trigger: "link",- };- })- .filter((c) => c !== null);-- // clickhouse only supports max 12 partitions (months) for a given event backfill- // so we need to transform this into a list of lists, one for each year- const clicksToCreateTB = clicksToCreate.reduce(- (acc, curr) => {- const year = new Date(curr.timestamp).getFullYear();- if (!acc[year]) {- acc[year] = [];- }- acc[year].push(curr);- return acc;- },- {} as Record<number, any[]>,- );-- // Record clicks- Object.entries(clicksToCreateTB).forEach(async ([year, clicks]) => {- const clicksBatch = clicks as typeof clicksToCreate;- console.log(`backfilling ${clicksBatch.length} clicks for ${year}`);- const clickRes = await fetch(- `${process.env.TINYBIRD_API_URL}/v0/events?name=dub_click_events&wait=true`,- {- method: "POST",- headers: {- Authorization: `Bearer ${process.env.TINYBIRD_API_KEY}`,- "Content-Type": "application/x-ndjson",- },- body: (clicksBatch as typeof clicksToCreate)- .map((d) => JSON.stringify(d))- .join("\n"),- },- ).then((res) => res.json());- console.log("backfilled clicks", JSON.stringify(clickRes, null, 2));- });-- const customersToCreate = finalLeadsToBackfill- .map((lead, idx) => {- const clickData = clicksToCreate[idx];- if (!clickData) {- return null;- }- return {- id: createId({ prefix: "cus_" }),- name: lead.customerExternalId,- email: lead.customerExternalId,- externalId: lead.customerExternalId,- projectId: workspace.id,- projectConnectId: workspace.stripeConnectId,- clickId: clickData.click_id,- linkId: clickData.link_id,- country: clickData.country,- clickedAt: new Date(lead.timestamp).toISOString(),- createdAt: new Date(lead.timestamp).toISOString(),- };- })- .filter((c) => c !== null);-- console.table(customersToCreate.slice(0, 10));-- const customerRes = await prisma.customer.createMany({- data: customersToCreate,- skipDuplicates: true,- });- console.log("backfilled customers", JSON.stringify(customerRes, null, 2));-- const leadsToCreate = clicksToCreate.map((clickData, idx) => ({- ...clickData,- event_id: nanoid(16),- event_name: "Sign up",- customer_id: customersToCreate[idx]!.id,- }));-- // same batching logic as above- const leadsToCreateTB = leadsToCreate.reduce(- (acc, curr) => {- const year = new Date(curr.timestamp).getFullYear();- if (!acc[year]) {- acc[year] = [];- }- acc[year].push(curr);- return acc;- },- {} as Record<number, any[]>,- );-- Object.entries(leadsToCreateTB).forEach(async ([year, leads]) => {- const leadsBatch = leads as typeof leadsToCreate;- console.log(`backfilling ${leadsBatch.length} leads for ${year}`);- const leadRes = await recordLeadWithTimestamp(leadsBatch);- console.log("backfilled leads", JSON.stringify(leadRes, null, 2));- });-- const statsByLink = finalLeadsToBackfill- .filter((lead) =>- partnerLinks.some(- (link) =>- link.key.toLowerCase() === lead.partnerLinkKey.toLowerCase(),- ),- )- .reduce(- (acc, lead) => {- const link = partnerLinks.find(- (link) =>- link.key.toLowerCase() === lead.partnerLinkKey.toLowerCase(),- )!;- const leadCreatedAt = new Date(lead.timestamp);- acc[link.id] = {- clicks: (acc[link.id]?.clicks || 0) + 1,- leads: (acc[link.id]?.leads || 0) + 1,- // if there is no lastLeadAt, or the leadCreatedAt is greater than the lastLeadAt, set the lastLeadAt to the leadCreatedAt- lastLeadAt:- leadCreatedAt > (acc[link.id]?.lastLeadAt ?? new Date(0))- ? leadCreatedAt- : acc[link.id]?.lastLeadAt,- };- return acc;- },- {} as Record<- string,- {- clicks: number;- leads: number;- lastLeadAt: Date | undefined;- }- >,- );-- console.log(- JSON.stringify(Object.entries(statsByLink).slice(0, 10), null, 2),- );-- for (const [linkId, stats] of Object.entries(statsByLink)) {- const res = await prisma.link.update({- where: { id: linkId },- data: {- clicks: {- increment: stats.clicks,- },- leads: {- increment: stats.leads,- },- lastLeadAt: stats.lastLeadAt,- },- });- console.log(- `Updated ${linkId} to ${res.clicks} clicks (+${stats.clicks} clicks), ${res.leads} leads (+${stats.leads} leads)`,- );- }- },- });-}--main();diff --git a/apps/web/scripts/testimonial/final-sync-commissions.ts b/apps/web/scripts/testimonial/final-sync-commissions.tsdeleted file mode 100644index 9d62862521..0000000000--- a/apps/web/scripts/testimonial/final-sync-commissions.ts+++ /dev/null@@ -1,64 +0,0 @@-import { prisma } from "@dub/prisma";-import { Prisma } from "@dub/prisma/client";-import "dotenv-flow/config";--// update commissions for a program-async function main() {- const where: Prisma.CommissionWhereInput = {- programId: "prog_xxx",- payoutId: {- not: null,- },- status: "processed",- };-- const payoutsStats = await prisma.commission.groupBy({- by: ["payoutId"],- where,- _count: true,- _sum: {- earnings: true,- },- orderBy: {- _sum: {- earnings: "desc",- },- },- });- console.log(payoutsStats);-- const actualPayouts = await prisma.payout.findMany({- where: {- id: {- in: payoutsStats.map((payout) => payout.payoutId!),- },- },- select: {- id: true,- amount: true,- },- });-- for (const payout of actualPayouts) {- const payoutStats = payoutsStats.find(- (payoutStat) => payoutStat.payoutId === payout.id,- );- if (!payoutStats) {- console.log(`Payout ${payout.id} not found in payoutsStats`);- continue;- }- if (payoutStats._sum.earnings !== payout.amount) {- console.log(- `Payout ${payout.id} has a mismatch, updating payout and commissions`,- );- await prisma.payout.update({- where: { id: payout.id },- data: {- amount: payoutStats._sum.earnings ?? 0,- },- });- }- }-}--main();diff --git a/apps/web/scripts/testimonial/sync-commissions.ts b/apps/web/scripts/testimonial/sync-commissions.tsdeleted file mode 100644index 437bcafe32..0000000000--- a/apps/web/scripts/testimonial/sync-commissions.ts+++ /dev/null@@ -1,72 +0,0 @@-import { prisma } from "@dub/prisma";-import { Prisma } from "@dub/prisma/client";-import "dotenv-flow/config";--// update commissions for a program-async function main() {- const where: Prisma.CommissionWhereInput = {- programId: "prog_xxx",- payoutId: {- not: null,- },- status: "paid",- };-- const payoutsToUpdate = await prisma.commission.groupBy({- by: ["payoutId"],- where,- _count: true,- orderBy: {- _count: {- payoutId: "desc",- },- },- });-- for (const payout of payoutsToUpdate) {- if (!payout.payoutId) {- console.log(`No payout ID found for payout ${payout.payoutId}`);- continue;- }-- await prisma.commission.updateMany({- where: {- payoutId: payout.payoutId,- status: "paid",- },- data: { payoutId: null },- });-- const remainingCommissions = await prisma.commission.findMany({- where: {- payoutId: payout.payoutId,- },- });-- console.log(- `Updated ${payout.payoutId} to have ${remainingCommissions.length} commissions`,- );-- if (remainingCommissions.length === 0) {- console.log(- `No remaining commissions for payout ${payout.payoutId}, deleting payout`,- );- await prisma.payout.delete({- where: { id: payout.payoutId },- });- continue;- }-- await prisma.payout.update({- where: { id: payout.payoutId },- data: {- amount: remainingCommissions.reduce(- (acc, commission) => acc + commission.earnings,- 0,- ),- },- });- }-}--main();diff --git a/apps/web/scripts/testimonial/update-commissions.ts b/apps/web/scripts/testimonial/update-commissions.tsdeleted file mode 100644index 7cf6752fc6..0000000000--- a/apps/web/scripts/testimonial/update-commissions.ts+++ /dev/null@@ -1,58 +0,0 @@-import { serializeReward } from "@/lib/api/partners/serialize-reward";-import { calculateSaleEarnings } from "@/lib/api/sales/calculate-sale-earnings";-import { prisma } from "@dub/prisma";-import { Prisma } from "@dub/prisma/client";-import "dotenv-flow/config";--// update commissions for a program-async function main() {- const where: Prisma.CommissionWhereInput = {- earnings: 0,- programId: "prog_xxx",- };-- const commissions = await prisma.commission.findMany({- where,- take: 100,- });-- const reward = await prisma.reward.findUniqueOrThrow({- where: {- id: "rw_xxx",- },- });-- const updatedCommissions = await Promise.all(- commissions.map(async (commission) => {- // Recalculate the earnings based on the new amount- const earnings = calculateSaleEarnings({- reward: serializeReward(reward),- sale: {- amount: commission.amount,- quantity: commission.quantity,- },- });-- return prisma.commission.update({- where: { id: commission.id },- data: {- earnings,- },- });- }),- );- console.table(updatedCommissions, [- "id",- "partnerId",- "amount",- "earnings",- "createdAt",- ]);-- const remainingCommissions = await prisma.commission.count({- where,- });- console.log(`${remainingCommissions} commissions left to update`);-}--main();