Skip to content

armgit / dub

Public Imported from GitHub
0

The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.

replace jev for embeddings

Pedro Ladeira88d74627cbf9
diff --git a/apps/web/.env.example b/apps/web/.env.example
index 5c26914520..ed3641a18a 100644
--- a/apps/web/.env.example
+++ b/apps/web/.env.example
@@ -153,6 +153,10 @@ PLAIN_WEBHOOK_SECRET=
UPSTASH_VECTOR_REST_URL=
UPSTASH_VECTOR_REST_TOKEN=
+# Emoji search index
+UPSTASH_VECTOR_EMOJI_REST_URL=
+UPSTASH_VECTOR_EMOJI_REST_TOKEN=
+
# Slack integration
SLACK_CLIENT_ID=
SLACK_CLIENT_SECRET=
diff --git a/apps/web/app/api/ai/emoji-search/route.ts b/apps/web/app/api/ai/emoji-search/route.ts
index 70e66fd0d6..c87d55a339 100644
--- a/apps/web/app/api/ai/emoji-search/route.ts
+++ b/apps/web/app/api/ai/emoji-search/route.ts
@@ -1,4 +1,4 @@
-import { searchEmojis, warmEmojiCatalog } from "@/lib/ai/search-emojis";
+import { searchEmojis } from "@/lib/ai/search-emojis";
import { withSession } from "@/lib/auth";
import { assertRateLimit } from "@/lib/upstash/assert-rate-limit";
import { RATELIMIT_POLICIES } from "@/lib/upstash/ratelimit-policies";
@@ -9,8 +9,6 @@ const emojiSearchSchema = z.object({
query: z.string().trim().min(2).max(64),
});
-warmEmojiCatalog();
-
// POST /api/ai/emoji-search – semantic emoji matches when name search misses
export const POST = withSession(async ({ req, session }) => {
const { query } = emojiSearchSchema.parse(await req.json());
diff --git a/apps/web/lib/ai/search-emojis.ts b/apps/web/lib/ai/search-emojis.ts
index f046efdc82..61c4629d66 100644
--- a/apps/web/lib/ai/search-emojis.ts
+++ b/apps/web/lib/ai/search-emojis.ts
@@ -1,145 +1,38 @@
-import { experimental_evaluate as evaluate } from "ai";
-
-const EMOJIBASE_DATA_URL =
- "https://cdn.jsdelivr.net/npm/[email protected]/en/data.json";
+import { getEmojiVectorIndex } from "@/lib/upstash/vector";
const MATCH_LIMIT = 18;
-const MATCH_THRESHOLD = 0.25;
-const MAX_QUESTIONS_JSON_CHARS = 100_000;
-const MAX_TAGS = 12;
-const SEARCH_CACHE_LIMIT = 100;
-
-const MATCH_INSTRUCTIONS =
- "Is this a good emoji for the search, including the same subject, synonyms, and close themes such as movies, characters, places, and objects?";
+const MATCH_SCORE_FLOOR = 0.45;
export type EmojiMatch = {
emoji: string;
label: string;
};
-export type EmojiCatalogEntry = EmojiMatch & {
- id: string;
- tags?: string[];
+type EmojiVectorHit = {
+ score: number;
+ metadata?: {
+ emoji?: string;
+ label?: string;
+ } | null;
};
-let catalogPromise: Promise<EmojiCatalogEntry[]> | null = null;
-const searchCache = new Map<string, EmojiMatch[]>();
-const searchRequests = new Map<string, Promise<EmojiMatch[]>>();
-
-export function rankEmojiMatches(
- catalog: EmojiCatalogEntry[],
- probabilities: Record<string, number>,
-): EmojiMatch[] {
- return catalog
- .flatMap((entry) => {
- const probability = probabilities[entry.id];
-
- if (probability === undefined || probability <= MATCH_THRESHOLD) {
- return [];
- }
-
- return [{ ...entry, probability }];
- })
- .sort((a, b) => b.probability - a.probability)
- .slice(0, MATCH_LIMIT)
- .map(({ emoji, label }) => ({ emoji, label }));
-}
-
-function emojiQuestion(entry: EmojiCatalogEntry) {
- const tags = (entry.tags ?? [])
- .filter(
- (tag) =>
- tag.length > 1 && tag.toLowerCase() !== entry.label.toLowerCase(),
- )
- .slice(0, MAX_TAGS);
-
- return {
- type: "boolean" as const,
- instructions: MATCH_INSTRUCTIONS,
- criteria: {
- true:
- tags.length === 0 ? entry.label : `${entry.label} (${tags.join(", ")})`,
- false: "unrelated subject",
- },
- };
-}
+export function toEmojiMatches(hits: EmojiVectorHit[]): EmojiMatch[] {
+ const matches: EmojiMatch[] = [];
-export function chunkEmojiCatalog(
- catalog: EmojiCatalogEntry[],
-): EmojiCatalogEntry[][] {
- const chunks: EmojiCatalogEntry[][] = [];
- let current: EmojiCatalogEntry[] = [];
- let size = 2;
+ for (const hit of hits) {
+ if (hit.score < MATCH_SCORE_FLOOR) continue;
- for (const entry of catalog) {
- const question = emojiQuestion(entry);
- const entrySize =
- JSON.stringify(entry.id).length + JSON.stringify(question).length + 1;
+ const emoji = hit.metadata?.emoji;
+ const label = hit.metadata?.label?.trim();
- if (current.length > 0 && size + entrySize > MAX_QUESTIONS_JSON_CHARS) {
- chunks.push(current);
- current = [];
- size = 2;
- }
+ if (!emoji || !label) continue;
- current.push(entry);
- size += entrySize;
- }
+ matches.push({ emoji, label });
- if (current.length > 0) {
- chunks.push(current);
+ if (matches.length >= MATCH_LIMIT) break;
}
- return chunks;
-}
-
-function getEmojiCatalog() {
- catalogPromise ??= fetch(EMOJIBASE_DATA_URL)
- .then(async (response) => {
- if (!response.ok) {
- throw new Error("Failed to load emoji data");
- }
-
- const data: unknown = await response.json();
-
- if (!Array.isArray(data)) {
- throw new Error("Failed to load emoji data");
- }
-
- const catalog: EmojiCatalogEntry[] = [];
- const seen = new Set<string>();
-
- for (const item of data) {
- if (!item || typeof item !== "object") continue;
-
- const record = item as Record<string, unknown>;
- const emoji = typeof record.emoji === "string" ? record.emoji : null;
- const label =
- typeof record.label === "string" ? record.label.trim() : null;
- const hexcode =
- typeof record.hexcode === "string" ? record.hexcode : null;
- const tags = Array.isArray(record.tags)
- ? record.tags.filter((tag): tag is string => typeof tag === "string")
- : [];
-
- if (!emoji || !label || !hexcode || seen.has(hexcode)) continue;
-
- seen.add(hexcode);
- catalog.push({ id: hexcode, emoji, label, tags });
- }
-
- return catalog;
- })
- .catch((error) => {
- catalogPromise = null;
- throw error;
- });
-
- return catalogPromise;
-}
-
-export function warmEmojiCatalog() {
- void getEmojiCatalog().catch(() => undefined);
+ return matches;
}
export async function searchEmojis(query: string): Promise<EmojiMatch[]> {
@@ -149,59 +42,11 @@ export async function searchEmojis(query: string): Promise<EmojiMatch[]> {
return [];
}
- const cached = searchCache.get(normalized);
- if (cached) return cached;
-
- const pending = searchRequests.get(normalized);
- if (pending) return pending;
-
- const request = getEmojiCatalog()
- .then(async (catalog) => {
- const probabilitySets = await Promise.all(
- chunkEmojiCatalog(catalog).map(async (entries) => {
- const result = await evaluate({
- model: "typesafe-ai/jev",
- state: normalized,
- questions: Object.fromEntries(
- entries.map((entry) => [entry.id, emojiQuestion(entry)]),
- ),
- providerOptions: {
- gateway: {
- zeroDataRetention: true,
- },
- },
- });
-
- const probabilities: Record<string, number> = {};
-
- for (const entry of entries) {
- const answer = result.answers[entry.id];
-
- if (answer?.type === "boolean") {
- probabilities[entry.id] = answer.probability;
- }
- }
-
- return probabilities;
- }),
- );
- const matches = rankEmojiMatches(
- catalog,
- Object.assign({}, ...probabilitySets),
- );
-
- if (searchCache.size >= SEARCH_CACHE_LIMIT) {
- const oldest = searchCache.keys().next().value;
- if (oldest) searchCache.delete(oldest);
- }
-
- searchCache.set(normalized, matches);
- return matches;
- })
- .finally(() => {
- searchRequests.delete(normalized);
- });
+ const hits = await getEmojiVectorIndex().query({
+ data: normalized,
+ topK: MATCH_LIMIT,
+ includeMetadata: true,
+ });
- searchRequests.set(normalized, request);
- return request;
+ return toEmojiMatches(hits);
}
diff --git a/apps/web/lib/upstash/vector.ts b/apps/web/lib/upstash/vector.ts
index 8a3ebec27c..1f54cff39b 100644
--- a/apps/web/lib/upstash/vector.ts
+++ b/apps/web/lib/upstash/vector.ts
@@ -4,3 +4,26 @@ export const vectorIndex = new Index({
url: process.env.UPSTASH_VECTOR_REST_URL!,
token: process.env.UPSTASH_VECTOR_REST_TOKEN!,
});
+
+type EmojiVectorMetadata = {
+ emoji: string;
+ label: string;
+};
+
+let emojiVectorIndex: Index<EmojiVectorMetadata> | null = null;
+
+export function getEmojiVectorIndex() {
+ if (emojiVectorIndex) return emojiVectorIndex;
+
+ const url = process.env.UPSTASH_VECTOR_EMOJI_REST_URL;
+ const token = process.env.UPSTASH_VECTOR_EMOJI_REST_TOKEN;
+
+ if (!url || !token) {
+ throw new Error(
+ "Set UPSTASH_VECTOR_EMOJI_REST_URL and UPSTASH_VECTOR_EMOJI_REST_TOKEN",
+ );
+ }
+
+ emojiVectorIndex = new Index<EmojiVectorMetadata>({ url, token });
+ return emojiVectorIndex;
+}
diff --git a/apps/web/scripts/dev/upsert-emoji-embeddings.ts b/apps/web/scripts/dev/upsert-emoji-embeddings.ts
new file mode 100644
index 0000000000..21151b0bf1
--- /dev/null
+++ b/apps/web/scripts/dev/upsert-emoji-embeddings.ts
@@ -0,0 +1,102 @@
+import { toEmojiMatches } from "@/lib/ai/search-emojis";
+import { getEmojiVectorIndex } from "@/lib/upstash/vector";
+import "dotenv-flow/config";
+
+const EMOJIBASE_DATA_URL =
+ "https://cdn.jsdelivr.net/npm/[email protected]/en/data.json";
+const UPSERT_BATCH_SIZE = 500;
+
+type EmojiVectorRecord = {
+ id: string;
+ data: string;
+ metadata: {
+ emoji: string;
+ label: string;
+ };
+};
+
+async function loadEmojiRecords(): Promise<EmojiVectorRecord[]> {
+ const response = await fetch(EMOJIBASE_DATA_URL, {
+ headers: { "User-Agent": "Mozilla/5.0" },
+ });
+
+ if (!response.ok) {
+ throw new Error("Failed to load emoji data");
+ }
+
+ const data: unknown = await response.json();
+
+ if (!Array.isArray(data)) {
+ throw new Error("Failed to load emoji data");
+ }
+
+ const records: EmojiVectorRecord[] = [];
+ const seen = new Set<string>();
+
+ for (const item of data) {
+ if (!item || typeof item !== "object") continue;
+
+ const record = item as Record<string, unknown>;
+ const emoji = typeof record.emoji === "string" ? record.emoji : null;
+ const label = typeof record.label === "string" ? record.label.trim() : null;
+ const hexcode = typeof record.hexcode === "string" ? record.hexcode : null;
+
+ if (!emoji || !label || !hexcode || seen.has(hexcode)) continue;
+
+ const tags = Array.isArray(record.tags)
+ ? record.tags.filter(
+ (tag): tag is string =>
+ typeof tag === "string" &&
+ tag.length > 1 &&
+ tag.toLowerCase() !== label.toLowerCase(),
+ )
+ : [];
+
+ seen.add(hexcode);
+ records.push({
+ id: hexcode,
+ data: tags.length === 0 ? label : `${label} (${tags.join(", ")})`,
+ metadata: { emoji, label },
+ });
+ }
+
+ return records;
+}
+
+async function main() {
+ if (
+ !process.env.UPSTASH_VECTOR_EMOJI_REST_URL ||
+ !process.env.UPSTASH_VECTOR_EMOJI_REST_TOKEN
+ ) {
+ throw new Error(
+ "Set UPSTASH_VECTOR_EMOJI_REST_URL and UPSTASH_VECTOR_EMOJI_REST_TOKEN",
+ );
+ }
+
+ const records = await loadEmojiRecords();
+
+ for (let index = 0; index < records.length; index += UPSERT_BATCH_SIZE) {
+ const batch = records.slice(index, index + UPSERT_BATCH_SIZE);
+ await getEmojiVectorIndex().upsert(batch);
+ console.log(
+ `Upserted ${Math.min(index + batch.length, records.length)}/${records.length}`,
+ );
+ }
+
+ const query = "jurassic park";
+ const hits = await getEmojiVectorIndex().query({
+ data: query,
+ topK: 18,
+ includeMetadata: true,
+ });
+
+ console.log(`\n${query}`);
+ for (const match of toEmojiMatches(hits)) {
+ console.log(`${match.emoji} ${match.label}`);
+ }
+}
+
+main().catch((error) => {
+ console.error(error);
+ process.exit(1);
+});
diff --git a/apps/web/tests/misc/search-emojis.test.ts b/apps/web/tests/misc/search-emojis.test.ts
deleted file mode 100644
index 4312097a75..0000000000
--- a/apps/web/tests/misc/search-emojis.test.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-import {
- chunkEmojiCatalog,
- rankEmojiMatches,
- type EmojiCatalogEntry,
-} from "@/lib/ai/search-emojis";
-import { describe, expect, it } from "vitest";
-
-function entry(
- id: string,
- emoji: string,
- label = "grinning face",
-): EmojiCatalogEntry {
- return { id, emoji, label };
-}
-
-describe("rankEmojiMatches", () => {
- const catalog = [
- entry("1", "😀", "grinning face"),
- entry("2", "🍕", "pizza"),
- entry("3", "🤒", "face with thermometer"),
- ];
-
- it("keeps thematic matches above 0.25, highest probability first", () => {
- expect(
- rankEmojiMatches(catalog, {
- "1": 0.3,
- "2": 0.25,
- "3": 0.91,
- }),
- ).toEqual([
- { emoji: "🤒", label: "face with thermometer" },
- { emoji: "😀", label: "grinning face" },
- ]);
- });
-
- it("returns at most 18 matches", () => {
- const many = Array.from({ length: 50 }, (_, index) =>
- entry(String(index), String.fromCodePoint(0x1f600 + (index % 20))),
- );
-
- const probabilities = Object.fromEntries(
- many.map((item, index) => [item.id, 0.7 + index / 1000]),
- );
-
- expect(rankEmojiMatches(many, probabilities)).toHaveLength(18);
- });
-});
-
-describe("chunkEmojiCatalog", () => {
- it("keeps a small catalog in one request", () => {
- expect(
- chunkEmojiCatalog([entry("1F600", "😀"), entry("1F355", "🍕", "pizza")]),
- ).toHaveLength(1);
- });
-
- it("splits a catalog that would exceed Jev's context window", () => {
- const label = "a".repeat(60_000);
-
- expect(
- chunkEmojiCatalog([entry("1", "😀", label), entry("2", "🍕", label)]),
- ).toHaveLength(2);
- });
-});