Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

add trigger-sync-embeddings script, remove old scripts

Steven Tey54b1a8ff70c7
diff --git a/apps/web/scripts/misc/cleanup-fraud-events.ts b/apps/web/scripts/misc/cleanup-fraud-events.ts
deleted file mode 100644
index fd25f700a7..0000000000
--- a/apps/web/scripts/misc/cleanup-fraud-events.ts
+++ /dev/null
@@ -1,88 +0,0 @@
-import "dotenv-flow/config";
-
-import { prisma } from "@/lib/prisma";
-
-async function main() {
- const fraudEvents = await prisma.fraudEvent.findMany({
- where: {
- fraudEventGroup: {
- type: "customerEmailMatch",
- resolvedAt: null,
- },
- metadata: {
- path: "$.matchType",
- equals: "historicalDomainMatch",
- },
- customer: {
- sales: {
- gt: 1,
- },
- },
- },
- include: {
- partner: true,
- customer: true,
- fraudEventGroup: {
- select: {
- program: true,
- },
- },
- },
- orderBy: {
- createdAt: "desc",
- },
- });
-
- console.table(
- fraudEvents.map((event) => ({
- id: event.id,
- program: event.fraudEventGroup?.program?.name,
- partner: event.partner?.email,
- customer: event.customer?.email,
- createdAt: event.createdAt,
- })),
- );
-
- // Remove fraud events
- if (fraudEvents.length > 0) {
- const { count } = await prisma.fraudEvent.deleteMany({
- where: {
- id: {
- in: fraudEvents.map((event) => event.id),
- },
- },
- });
-
- console.log(`Removed ${count} fraud events`);
- }
-
- // Remove fraud group if no events left
- const fraudEventGroupsWithNoEvents = await prisma.fraudEventGroup.findMany({
- where: {
- fraudEvents: {
- none: {},
- },
- },
- select: {
- id: true,
- },
- });
-
- console.log(
- `Found ${fraudEventGroupsWithNoEvents.length} with no events left.`,
- );
-
- if (fraudEventGroupsWithNoEvents.length > 0) {
- const { count } = await prisma.fraudEventGroup.deleteMany({
- where: {
- id: {
- in: fraudEventGroupsWithNoEvents.map((group) => group.id),
- },
- },
- });
-
- console.log(`Removed ${count} fraud event groups`);
- }
-}
-
-main();
diff --git a/apps/web/scripts/misc/remove-program-ban-fraud-events.ts b/apps/web/scripts/misc/remove-program-ban-fraud-events.ts
deleted file mode 100644
index 778f72d2c6..0000000000
--- a/apps/web/scripts/misc/remove-program-ban-fraud-events.ts
+++ /dev/null
@@ -1,52 +0,0 @@
-import { prisma } from "@/lib/prisma";
-import "dotenv-flow/config";
-
-// script to remove fraud events for cross-program bans
-async function main() {
- const fraudEventsToRemove = await prisma.fraudEvent.findMany({
- where: {
- fraudEventGroup: {
- type: "partnerCrossProgramBan",
- status: "pending",
- },
- },
- });
-
- console.log(`Found ${fraudEventsToRemove.length} fraud events to remove`);
-
- const deleted = await prisma.fraudEvent.deleteMany({
- where: {
- id: {
- in: fraudEventsToRemove.map((event) => event.id),
- },
- },
- });
-
- console.log(`Removed ${deleted.count} fraud events`);
-
- const fraudEventGroupsWithNoEvents = await prisma.fraudEventGroup.findMany({
- where: {
- type: "partnerCrossProgramBan",
- status: "pending",
- fraudEvents: {
- none: {},
- },
- },
- });
-
- console.log(
- `Found ${fraudEventGroupsWithNoEvents.length} fraud event groups with no events`,
- );
-
- const deletedGroups = await prisma.fraudEventGroup.deleteMany({
- where: {
- id: {
- in: fraudEventGroupsWithNoEvents.map((group) => group.id),
- },
- },
- });
-
- console.log(`Removed ${deletedGroups.count} fraud event groups`);
-}
-
-main();
diff --git a/apps/web/scripts/misc/restore-link-analytics.ts b/apps/web/scripts/misc/restore-link-analytics.ts
deleted file mode 100644
index fa86653936..0000000000
--- a/apps/web/scripts/misc/restore-link-analytics.ts
+++ /dev/null
@@ -1,78 +0,0 @@
-import { prisma } from "@/lib/prisma";
-import { chunk, parseFilterValue } from "@dub/utils";
-import "dotenv-flow/config";
-import * as fs from "fs";
-import * as Papa from "papaparse";
-import { getAnalytics } from "../../lib/analytics/get-analytics";
-
-let linksToRestore: { id: string }[] = [];
-
-async function main() {
- Papa.parse(fs.createReadStream("deleted_links.csv", "utf-8"), {
- header: true,
- skipEmptyLines: true,
- step: (result: {
- data: {
- link_id: string;
- };
- }) => {
- linksToRestore.push({ id: result.data.link_id });
- },
- complete: async () => {
- console.log(`Found ${linksToRestore.length} links to restore`);
-
- const chunks = chunk(linksToRestore, 200);
- for (let i = 0; i < chunks.length; i++) {
- const chunk = chunks[i];
- const linkIds = chunk.map((link) => link.id!);
- const stats = await getAnalytics({
- event: "composite",
- groupBy: "top_links",
- linkId: parseFilterValue(linkIds),
- interval: "1y",
- });
- if (stats.length === 0) {
- console.log(
- `No stats found for links in batch ${i + 1} of ${chunks.length}`,
- );
- continue;
- }
- console.log(
- `Stats found for links in batch ${i + 1} of ${chunks.length}`,
- );
-
- const linksToBackfill = stats.map((stat) => ({
- id: stat.id,
- clicks: stat.clicks,
- leads: stat.leads,
- conversions: Math.min(stat.leads, stat.sales),
- sales: stat.sales,
- saleAmount: stat.saleAmount,
- }));
- await Promise.all(
- linksToBackfill.map(async (link) => {
- const res = await prisma.link.update({
- where: { id: link.id },
- data: {
- clicks: link.clicks,
- leads: link.leads,
- conversions: link.conversions,
- sales: link.sales,
- saleAmount: link.saleAmount,
- },
- });
- console.log(
- `Updated ${link.id} to ${res.clicks} clicks, ${res.leads} leads, ${res.conversions} conversions, ${res.sales} sales, ${res.saleAmount} saleAmount`,
- );
- }),
- );
-
- console.log(
- `Backfilled stats for ${linksToBackfill.length} links in batch ${i + 1} of ${chunks.length}`,
- );
- }
- },
- });
-}
-
-main();
diff --git a/apps/web/scripts/misc/restore-links.ts b/apps/web/scripts/misc/restore-links.ts
deleted file mode 100644
index c82709a6f9..0000000000
--- a/apps/web/scripts/misc/restore-links.ts
+++ /dev/null
@@ -1,95 +0,0 @@
-import { prisma } from "@/lib/prisma";
-import { chunk, getParamsFromURL, linkConstructorSimple } from "@dub/utils";
-import { Prisma } from "@prisma/client";
-import "dotenv-flow/config";
-import * as fs from "fs";
-import * as Papa from "papaparse";
-
-let linksToRestore: Prisma.LinkCreateManyInput[] = [];
-let linkTagsToRestore: Prisma.LinkTagCreateManyInput[] = [];
-
-async function main() {
- Papa.parse(fs.createReadStream("deleted_links.csv", "utf-8"), {
- header: true,
- skipEmptyLines: true,
- step: (result: {
- data: {
- link_id: string;
- domain: string;
- key: string;
- url: string;
- tag_ids: string;
- workspace_id: string;
- created_at: string;
- program_id: string;
- partner_id: string;
- folder_id: string;
- };
- }) => {
- const userId = "user_xxx";
- if (!userId) {
- throw new Error("No user id found for link");
- }
- const { utm_source, utm_medium, utm_campaign, utm_term, utm_content } =
- getParamsFromURL(result.data.url);
- const tagIds: string[] = result.data.tag_ids
- ? JSON.parse(result.data.tag_ids.replace(/'/g, '"'))
- : [];
-
- linksToRestore.push({
- id: result.data.link_id,
- domain: result.data.domain,
- key: result.data.key,
- url: result.data.url,
- shortLink: linkConstructorSimple({
- domain: result.data.domain,
- key: result.data.key,
- }),
- projectId: result.data.workspace_id,
- createdAt: new Date(result.data.created_at),
- programId: result.data.program_id || null,
- partnerId: result.data.partner_id || null,
- folderId: result.data.folder_id || null,
- utm_source,
- utm_medium,
- utm_campaign,
- utm_term,
- utm_content,
- userId,
- trackConversion: true,
- });
-
- if (tagIds.length > 0) {
- tagIds.forEach((tagId) => {
- linkTagsToRestore.push({
- linkId: result.data.link_id,
- tagId,
- });
- });
- }
- },
- complete: async () => {
- console.log(`Found ${linksToRestore.length} links to restore`);
-
- const chunks = chunk(linksToRestore, 500);
- for (const chunk of chunks) {
- const res = await prisma.link.createMany({
- data: chunk,
- skipDuplicates: true,
- });
- console.log(`Created ${res.count} links (out of ${chunk.length})`);
- }
-
- const linkTagsChunks = chunk(linkTagsToRestore, 500);
- for (const chunk of linkTagsChunks) {
- const res = await prisma.linkTag.createMany({
- data: chunk,
- skipDuplicates: true,
- });
- console.log(`Created ${res.count} link tags (out of ${chunk.length})`);
- }
- },
- });
-}
-
-main();
diff --git a/apps/web/scripts/misc/restore-program-folders.ts b/apps/web/scripts/misc/restore-program-folders.ts
deleted file mode 100644
index 8d15fb91aa..0000000000
--- a/apps/web/scripts/misc/restore-program-folders.ts
+++ /dev/null
@@ -1,94 +0,0 @@
-import { includeProgramEnrollment } from "@/lib/api/links/include-program-enrollment";
-import { includeTags } from "@/lib/api/links/include-tags";
-import { prisma } from "@/lib/prisma";
-import "dotenv-flow/config";
-import { recordLink } from "../../lib/tinybird/record-link";
-
-async function main() {
- const deactivatedPrograms = await prisma.program.findMany({
- where: {
- deactivatedAt: {
- not: null,
- },
- },
- select: {
- id: true,
- slug: true,
- workspaceId: true,
- defaultFolderId: true,
- workspace: {
- select: {
- id: true,
- users: {
- select: {
- id: true,
- },
- },
- },
- },
- },
- });
-
- for (const program of deactivatedPrograms) {
- if (program.workspace.users.length === 0) {
- console.log(`No users found for workspace ${program.workspaceId}`);
- continue;
- }
-
- await prisma.folder.upsert({
- where: {
- id: program.defaultFolderId,
- },
- create: {
- id: program.defaultFolderId,
- name: "Partner Links",
- projectId: program.workspaceId,
- accessLevel: "write",
- users: {
- create: {
- userId: program.workspace.users[0].id,
- role: "owner",
- },
- },
- },
- update: {},
- });
-
- while (true) {
- const linksToUpdate = await prisma.link.findMany({
- where: {
- programId: program.id,
- folderId: null,
- },
- take: 250,
- });
-
- if (linksToUpdate.length === 0) {
- break;
- }
-
- const res = await prisma.link.updateMany({
- where: {
- id: { in: linksToUpdate.map((link) => link.id) },
- },
- data: { folderId: program.defaultFolderId },
- });
- console.log(`Updated ${res.count} links`);
-
- const updatedLinks = await prisma.link.findMany({
- where: {
- id: { in: linksToUpdate.map((link) => link.id) },
- },
- include: {
- ...includeTags,
- ...includeProgramEnrollment,
- },
- });
-
- const tbRes = await recordLink(updatedLinks);
- console.log("tbRes", tbRes);
- }
- }
-}
-
-main();
diff --git a/apps/web/scripts/misc/trigger-sync-embeddings.ts b/apps/web/scripts/misc/trigger-sync-embeddings.ts
new file mode 100644
index 0000000000..7be8e33a5e
--- /dev/null
+++ b/apps/web/scripts/misc/trigger-sync-embeddings.ts
@@ -0,0 +1,47 @@
+import "dotenv-flow/config";
+
+import { qstash } from "@/lib/cron";
+import { crawlSitemapUrls } from "@/lib/sitemaps/import-tracked-sitemaps";
+
+const DOCS_SITEMAP_URL = "https://dub.co/docs/sitemap.xml";
+const FLOW_CONTROL_KEY = "sync-docs-embeddings";
+
+// script to trigger a sync of all docs/help articles
+async function main() {
+ const secret = process.env.CRON_SECRET;
+ if (!secret) {
+ throw new Error("CRON_SECRET is not set");
+ }
+
+ console.log(`Fetching sitemap from ${DOCS_SITEMAP_URL}...`);
+ const { urls, hadErrors } = await crawlSitemapUrls(DOCS_SITEMAP_URL);
+
+ if (hadErrors) {
+ throw new Error(`Failed to fetch sitemap ${DOCS_SITEMAP_URL}`);
+ }
+
+ console.log(`Found ${urls.length} URLs to sync`);
+
+ for (const url of urls) {
+ const normalizedUrl = new URL(url).toString();
+
+ const response = await qstash.publishJSON({
+ url: "https://app.dub.co/api/ai/sync-embeddings",
+ method: "POST",
+ headers: { Authorization: `Bearer ${secret}` },
+ body: { url: normalizedUrl },
+ flowControl: {
+ key: FLOW_CONTROL_KEY,
+ rate: 1,
+ },
+ });
+
+ console.log(
+ `Queued sync for ${normalizedUrl} (messageId: ${response.messageId})`,
+ );
+ }
+
+ console.log(`Queued ${urls.length} embedding sync jobs`);
+}
+
+main();