Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

Support provider-aware partner search index management

Rafi1e117565ff22
diff --git a/apps/web/lib/api/partners/search/provider.ts b/apps/web/lib/api/partners/search/provider.ts
index eef9233cb0..47a2bd01f1 100644
--- a/apps/web/lib/api/partners/search/provider.ts
+++ b/apps/web/lib/api/partners/search/provider.ts
@@ -5,13 +5,29 @@ import type { PartnerSearchProvider } from "./types";
let upstashRedisProvider: PartnerSearchProvider | null = null;
let upstashSearchProvider: PartnerSearchProvider | null = null;
-export function getPartnerSearchProvider(): PartnerSearchProvider | null {
+export type PartnerSearchProviderName = "upstash-redis" | "upstash-search";
+
+export function getPartnerSearchProviderName(): PartnerSearchProviderName | null {
const provider = process.env.PARTNER_SEARCH_PROVIDER?.trim();
if (!provider) {
return null;
}
+ if (provider === "upstash-redis" || provider === "upstash-search") {
+ return provider;
+ }
+
+ throw new Error(`Unsupported partner search provider: ${provider}`);
+}
+
+export function getPartnerSearchProvider(): PartnerSearchProvider | null {
+ const provider = getPartnerSearchProviderName();
+
+ if (!provider) {
+ return null;
+ }
+
if (provider === "upstash-redis") {
if (!upstashRedisProvider) {
upstashRedisProvider = createUpstashRedisPartnerSearchProvider();
diff --git a/apps/web/lib/api/partners/search/providers/upstash-search.ts b/apps/web/lib/api/partners/search/providers/upstash-search.ts
index c73ab1b49f..5fd79f70d1 100644
--- a/apps/web/lib/api/partners/search/providers/upstash-search.ts
+++ b/apps/web/lib/api/partners/search/providers/upstash-search.ts
@@ -66,6 +66,7 @@ interface UpstashSearchIndexClient {
): Promise<string>;
delete(documentIds: string[]): Promise<{ deleted: number }>;
info(): Promise<{ pendingDocumentCount: number; documentCount: number }>;
+ reset(): Promise<string>;
}
interface CreateUpstashSearchPartnerSearchProviderOptions {
@@ -274,6 +275,22 @@ function logPartnerSearchDebug(
console.log(`[Partner Search Debug] Upstash Search ${operation}`, details);
}
+export async function resetUpstashSearchPartnerSearchIndex({
+ searchIndex,
+ indexName,
+}: CreateUpstashSearchPartnerSearchProviderOptions = {}) {
+ const resolvedIndexName = getIndexName(indexName);
+ const index = searchIndex ?? createSearchIndex(resolvedIndexName);
+ const { documentCount } = await withTransientRetry(() => index.info());
+
+ // Upstash Search reset clears the whole index in one operation. It is the
+ // appropriate way to remove documents from an obsolete index shape before
+ // a full backfill, instead of issuing one delete request per document.
+ await withTransientRetry(() => index.reset());
+
+ return { indexName: resolvedIndexName, documentCount };
+}
+
export function createUpstashSearchPartnerSearchProvider({
searchIndex,
indexName,
diff --git a/apps/web/scripts/dev/benchmark-partner-search.ts b/apps/web/scripts/dev/benchmark-partner-search.ts
index a75652dbd4..d893e8b29e 100644
--- a/apps/web/scripts/dev/benchmark-partner-search.ts
+++ b/apps/web/scripts/dev/benchmark-partner-search.ts
@@ -1,6 +1,8 @@
import { getPartners } from "@/lib/api/partners/get-partners";
import {
getPartnerSearchProvider,
+ getPartnerSearchProviderName,
+ PARTNER_SEARCH_CANDIDATE_LIMIT,
partnerSearchDocumentSelect,
serializePartnerSearchDocument,
type PartnerSearchDocument,
@@ -89,8 +91,10 @@ function parseArguments(args: string[]): BenchmarkArguments {
if (concurrency > requests) {
throw new Error("--concurrency cannot exceed --requests.");
}
- if (pageSize > 100) {
- throw new Error("--pageSize cannot exceed 100.");
+ if (pageSize > PARTNER_SEARCH_CANDIDATE_LIMIT) {
+ throw new Error(
+ `--pageSize cannot exceed ${PARTNER_SEARCH_CANDIDATE_LIMIT}.`,
+ );
}
return {
@@ -235,6 +239,7 @@ async function runWithConcurrency<T>(
async function main() {
const options = parseArguments(process.argv.slice(2));
const searchProvider = getPartnerSearchProvider();
+ const providerName = getPartnerSearchProviderName();
if (!searchProvider) {
throw new Error(
@@ -287,7 +292,8 @@ async function main() {
};
console.log(`Partner search benchmark for program ${options.programId}`);
- console.log(`${partnerCount.toLocaleString()} indexed partners`);
+ console.log(`Provider: ${providerName}`);
+ console.log(`${partnerCount.toLocaleString()} program partners`);
console.log(
`${options.requests.toLocaleString()} measured requests, ${options.warmupRequests.toLocaleString()} warm-up requests, concurrency ${options.concurrency}`,
);
diff --git a/apps/web/scripts/dev/debug-partner-search.ts b/apps/web/scripts/dev/debug-partner-search.ts
index de6d4ca948..11dde7ee8a 100644
--- a/apps/web/scripts/dev/debug-partner-search.ts
+++ b/apps/web/scripts/dev/debug-partner-search.ts
@@ -1,13 +1,13 @@
import {
getPartnerSearchableValues,
getPartnerSearchProvider,
+ getPartnerSearchProviderName,
normalizePartnerSearchQuery,
PARTNER_SEARCH_CANDIDATE_LIMIT,
partnerSearchDocumentSelect,
serializePartnerSearchDocument,
type PartnerSearchDocument,
type PartnerSearchHit,
- type PartnerSearchSortField,
} from "@/lib/api/partners/search";
import { prisma } from "@/lib/prisma";
import { parsePositiveInteger } from "@/scripts/utils/parse-positive-integer";
@@ -15,28 +15,12 @@ import { ProgramEnrollmentStatus } from "@prisma/client";
import "dotenv-flow/config";
const DEFAULT_LIMIT = 10;
-const MAX_LIMIT = 100;
-const SORT_FIELDS: PartnerSearchSortField[] = [
- "createdAt",
- "totalClicks",
- "totalLeads",
- "totalConversions",
- "totalSaleAmount",
- "totalCommissions",
- "netRevenue",
- "earningsPerClick",
- "averageLifetimeValue",
- "clickToLeadRate",
- "clickToConversionRate",
- "leadToConversionRate",
- "returnOnAdSpend",
-];
+const MAX_LIMIT = PARTNER_SEARCH_CANDIDATE_LIMIT;
interface DebugArguments {
programId: string;
query: string;
limit: number;
- sortBy: PartnerSearchSortField;
status?: ProgramEnrollmentStatus;
}
@@ -44,7 +28,6 @@ function parseArguments(args: string[]): DebugArguments {
let programId: string | undefined;
let query: string | undefined;
let limit = DEFAULT_LIMIT;
- let sortBy: PartnerSearchSortField = "totalSaleAmount";
let status: ProgramEnrollmentStatus | undefined;
for (const arg of args) {
@@ -54,12 +37,6 @@ function parseArguments(args: string[]): DebugArguments {
query = arg.slice("--query=".length);
} else if (arg.startsWith("--limit=")) {
limit = parsePositiveInteger(arg.slice("--limit=".length), "--limit");
- } else if (arg.startsWith("--sortBy=")) {
- const value = arg.slice("--sortBy=".length) as PartnerSearchSortField;
- if (!SORT_FIELDS.includes(value)) {
- throw new Error(`--sortBy must be one of: ${SORT_FIELDS.join(", ")}.`);
- }
- sortBy = value;
} else if (arg.startsWith("--status=")) {
const value = arg.slice("--status=".length) as ProgramEnrollmentStatus;
if (!Object.values(ProgramEnrollmentStatus).includes(value)) {
@@ -83,7 +60,7 @@ function parseArguments(args: string[]): DebugArguments {
throw new Error(`--limit cannot exceed ${MAX_LIMIT}.`);
}
- return { programId, query, limit, sortBy, status };
+ return { programId, query, limit, status };
}
async function getDatabaseDocuments(
@@ -122,13 +99,11 @@ function reportResults({
label,
hits,
databaseDocuments,
- sortBy,
normalizedQuery,
}: {
label: string;
hits: PartnerSearchHit[];
databaseDocuments: Awaited<ReturnType<typeof getDatabaseDocuments>>;
- sortBy: PartnerSearchSortField;
normalizedQuery: string;
}) {
console.log(`\n${label}`);
@@ -139,7 +114,6 @@ function reportResults({
return {
rank: index + 1,
providerScore: hit.score,
- databaseSortValue: databaseDocument?.[sortBy] ?? null,
containsLiteralQuery: containsLiteralQuery(
databaseDocument,
normalizedQuery,
@@ -155,7 +129,7 @@ function reportResults({
}
async function main() {
- const { programId, query, limit, sortBy, status } = parseArguments(
+ const { programId, query, limit, status } = parseArguments(
process.argv.slice(2),
);
const searchProvider = getPartnerSearchProvider();
@@ -163,7 +137,7 @@ async function main() {
throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");
}
- const providerName = process.env.PARTNER_SEARCH_PROVIDER?.trim();
+ const providerName = getPartnerSearchProviderName();
const startedAt = performance.now();
// Step 1: Fetch the same relevance candidates used by the website
@@ -202,7 +176,6 @@ async function main() {
label: "Provider relevance order",
hits: filteredHits,
databaseDocuments,
- sortBy,
normalizedQuery: normalizePartnerSearchQuery(query),
});
}
diff --git a/apps/web/scripts/partners/backfill-partner-search.ts b/apps/web/scripts/partners/backfill-partner-search.ts
index b320067885..83a7d29be3 100644
--- a/apps/web/scripts/partners/backfill-partner-search.ts
+++ b/apps/web/scripts/partners/backfill-partner-search.ts
@@ -1,5 +1,7 @@
import {
backfillPartnerSearch,
+ createUpstashRedisPartnerSearchIndex,
+ getPartnerSearchProviderName,
type PartnerSearchBackfillProgress,
} from "@/lib/api/partners/search";
import { prisma } from "@/lib/prisma";
@@ -62,8 +64,19 @@ function reportProgress({
async function main() {
const { programId, batchSize, after } = parseArguments(process.argv.slice(2));
+ const providerName = getPartnerSearchProviderName();
+ if (!providerName) {
+ throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");
+ }
+
+ // Redis Search indexes have an explicit schema and must exist before the
+ // backfill starts. Upstash Search creates its index on the first upsert.
+ if (providerName === "upstash-redis") {
+ await createUpstashRedisPartnerSearchIndex();
+ }
console.log(`Starting partner search backfill for program ${programId}`);
+ console.log(`Provider: ${providerName}`);
console.log(
`Batch size: ${batchSize.toLocaleString()}${after ? `, resuming after ${after}` : ""}`,
);
diff --git a/apps/web/scripts/partners/create-partner-search-index.ts b/apps/web/scripts/partners/create-partner-search-index.ts
index 2ac281af5a..a98a8931b4 100644
--- a/apps/web/scripts/partners/create-partner-search-index.ts
+++ b/apps/web/scripts/partners/create-partner-search-index.ts
@@ -1,7 +1,22 @@
-import { createUpstashRedisPartnerSearchIndex } from "@/lib/api/partners/search";
+import {
+ createUpstashRedisPartnerSearchIndex,
+ getPartnerSearchProviderName,
+} from "@/lib/api/partners/search";
import "dotenv-flow/config";
async function main() {
+ const providerName = getPartnerSearchProviderName();
+ if (!providerName) {
+ throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");
+ }
+
+ if (providerName === "upstash-search") {
+ console.log(
+ "Upstash Search creates the partner search index on the first backfill upsert.",
+ );
+ return;
+ }
+
const index = await createUpstashRedisPartnerSearchIndex();
const description = await index.describe();
diff --git a/apps/web/scripts/partners/delete-partner-search-index.ts b/apps/web/scripts/partners/delete-partner-search-index.ts
index 57f4390fef..a0446d2813 100644
--- a/apps/web/scripts/partners/delete-partner-search-index.ts
+++ b/apps/web/scripts/partners/delete-partner-search-index.ts
@@ -1,3 +1,7 @@
+import {
+ getPartnerSearchProviderName,
+ resetUpstashSearchPartnerSearchIndex,
+} from "@/lib/api/partners/search";
import { chunk } from "@dub/utils";
import { Redis } from "@upstash/redis";
import "dotenv-flow/config";
@@ -96,19 +100,35 @@ async function deleteDocuments(redis: Redis, indexName: string) {
async function main() {
const { indexName } = parseArguments(process.argv.slice(2));
+ const providerName = getPartnerSearchProviderName();
+ if (!providerName) {
+ throw new Error("PARTNER_SEARCH_PROVIDER is not configured.");
+ }
+
+ if (providerName === "upstash-search") {
+ const { documentCount } = await resetUpstashSearchPartnerSearchIndex({
+ indexName,
+ });
+ console.log(
+ `Partner search cleanup complete: ${documentCount.toLocaleString()} documents removed from ${indexName}.`,
+ );
+ return;
+ }
+
const redis = createRedisClient();
const index = redis.search.index({ name: indexName });
const description = await index.describe();
- const expectedPrefix = `${indexName}:`;
+ const expectedPrefixes = [`${indexName}:`, `${indexName}:partner:`];
- // Step 1: Verify that an existing index owns only the requested key prefix
+ // Step 1: Accept the earlier broad prefix and the current lean prefix. Both
+ // are contained by the index namespace deleted below.
if (
description &&
(description.prefixes.length !== 1 ||
- description.prefixes[0] !== expectedPrefix)
+ !expectedPrefixes.includes(description.prefixes[0]))
) {
throw new Error(
- `Index ${indexName} does not use the expected ${expectedPrefix} prefix.`,
+ `Index ${indexName} does not use an expected partner search prefix.`,
);
}
@@ -120,7 +140,8 @@ async function main() {
: `Partner search index ${indexName} did not exist`,
);
- // Step 3: Delete partner and tag documents stored under this index namespace
+ // Step 3: Delete current partner documents and any obsolete documents left
+ // under the same index namespace by an earlier index shape
const deleted = await deleteDocuments(redis, indexName);
console.log(
`Partner search cleanup complete: ${deleted.toLocaleString()} documents removed.`,
diff --git a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
index 7dd39c58b5..172d951912 100644
--- a/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
+++ b/apps/web/tests/partners/upstash-search-partner-search-provider.test.ts
@@ -1,5 +1,6 @@
import {
createUpstashSearchPartnerSearchProvider,
+ resetUpstashSearchPartnerSearchIndex,
type PartnerSearchDocument,
} from "@/lib/api/partners/search";
import { beforeEach, describe, expect, it, vi } from "vitest";
@@ -43,6 +44,7 @@ const document: PartnerSearchDocument = {
const mocks = vi.hoisted(() => ({
delete: vi.fn(),
info: vi.fn(),
+ reset: vi.fn(),
search: vi.fn(),
upsert: vi.fn(),
}));
@@ -51,6 +53,7 @@ function createSearchIndexMock() {
return {
delete: mocks.delete,
info: mocks.info,
+ reset: mocks.reset,
search: mocks.search,
upsert: mocks.upsert,
};
@@ -91,6 +94,7 @@ describe("Upstash Search partner search provider", () => {
documentCount: 0,
pendingDocumentCount: 0,
});
+ mocks.reset.mockResolvedValue("Success");
mocks.search.mockResolvedValue([]);
mocks.upsert.mockResolvedValue("Success");
});
@@ -272,4 +276,22 @@ describe("Upstash Search partner search provider", () => {
vi.useRealTimers();
}
});
+
+ it("resets every document in the index in one operation", async () => {
+ mocks.info.mockResolvedValue({
+ documentCount: 100_005,
+ pendingDocumentCount: 0,
+ });
+
+ await expect(
+ resetUpstashSearchPartnerSearchIndex({
+ searchIndex: createSearchIndexMock(),
+ indexName: "test-index",
+ }),
+ ).resolves.toEqual({
+ indexName: "test-index",
+ documentCount: 100_005,
+ });
+ expect(mocks.reset).toHaveBeenCalledOnce();
+ });
});