The modern link attribution platform. Loved by world-class engineering teams at Twilio, Zillow, Jobber, Vercel, Raycast, and more.
TypeScript clickhousemysqlnextjsopen-sourceplanetscaleprismatailwindcsstinybirdtypescriptupstashvercel
replace jev for embeddings
Pedro Ladeira88d74627cbf9
diff --git a/apps/web/.env.example b/apps/web/.env.exampleindex 5c26914520..ed3641a18a 100644--- a/apps/web/.env.example+++ b/apps/web/.env.example@@ -153,6 +153,10 @@ PLAIN_WEBHOOK_SECRET=UPSTASH_VECTOR_REST_URL=UPSTASH_VECTOR_REST_TOKEN=+# Emoji search index+UPSTASH_VECTOR_EMOJI_REST_URL=+UPSTASH_VECTOR_EMOJI_REST_TOKEN=+# Slack integrationSLACK_CLIENT_ID=SLACK_CLIENT_SECRET=diff --git a/apps/web/app/api/ai/emoji-search/route.ts b/apps/web/app/api/ai/emoji-search/route.tsindex 70e66fd0d6..c87d55a339 100644--- a/apps/web/app/api/ai/emoji-search/route.ts+++ b/apps/web/app/api/ai/emoji-search/route.ts@@ -1,4 +1,4 @@-import { searchEmojis, warmEmojiCatalog } from "@/lib/ai/search-emojis";+import { searchEmojis } from "@/lib/ai/search-emojis";import { withSession } from "@/lib/auth";import { assertRateLimit } from "@/lib/upstash/assert-rate-limit";import { RATELIMIT_POLICIES } from "@/lib/upstash/ratelimit-policies";@@ -9,8 +9,6 @@ const emojiSearchSchema = z.object({query: z.string().trim().min(2).max(64),});-warmEmojiCatalog();-// POST /api/ai/emoji-search – semantic emoji matches when name search missesexport const POST = withSession(async ({ req, session }) => {const { query } = emojiSearchSchema.parse(await req.json());diff --git a/apps/web/lib/ai/search-emojis.ts b/apps/web/lib/ai/search-emojis.tsindex f046efdc82..61c4629d66 100644--- a/apps/web/lib/ai/search-emojis.ts+++ b/apps/web/lib/ai/search-emojis.ts@@ -1,145 +1,38 @@-import { experimental_evaluate as evaluate } from "ai";--const EMOJIBASE_DATA_URL =- "https://cdn.jsdelivr.net/npm/[email protected]/en/data.json";+import { getEmojiVectorIndex } from "@/lib/upstash/vector";const MATCH_LIMIT = 18;-const MATCH_THRESHOLD = 0.25;-const MAX_QUESTIONS_JSON_CHARS = 100_000;-const MAX_TAGS = 12;-const SEARCH_CACHE_LIMIT = 100;--const MATCH_INSTRUCTIONS =- "Is this a good emoji for the search, including the same subject, synonyms, and close themes such as movies, characters, places, and objects?";+const MATCH_SCORE_FLOOR = 0.45;export type EmojiMatch = {emoji: string;label: string;};-export type EmojiCatalogEntry = EmojiMatch & {- id: string;- tags?: string[];+type EmojiVectorHit = {+ score: number;+ metadata?: {+ emoji?: string;+ label?: string;+ } | null;};-let catalogPromise: Promise<EmojiCatalogEntry[]> | null = null;-const searchCache = new Map<string, EmojiMatch[]>();-const searchRequests = new Map<string, Promise<EmojiMatch[]>>();--export function rankEmojiMatches(- catalog: EmojiCatalogEntry[],- probabilities: Record<string, number>,-): EmojiMatch[] {- return catalog- .flatMap((entry) => {- const probability = probabilities[entry.id];-- if (probability === undefined || probability <= MATCH_THRESHOLD) {- return [];- }-- return [{ ...entry, probability }];- })- .sort((a, b) => b.probability - a.probability)- .slice(0, MATCH_LIMIT)- .map(({ emoji, label }) => ({ emoji, label }));-}--function emojiQuestion(entry: EmojiCatalogEntry) {- const tags = (entry.tags ?? [])- .filter(- (tag) =>- tag.length > 1 && tag.toLowerCase() !== entry.label.toLowerCase(),- )- .slice(0, MAX_TAGS);-- return {- type: "boolean" as const,- instructions: MATCH_INSTRUCTIONS,- criteria: {- true:- tags.length === 0 ? entry.label : `${entry.label} (${tags.join(", ")})`,- false: "unrelated subject",- },- };-}+export function toEmojiMatches(hits: EmojiVectorHit[]): EmojiMatch[] {+ const matches: EmojiMatch[] = [];-export function chunkEmojiCatalog(- catalog: EmojiCatalogEntry[],-): EmojiCatalogEntry[][] {- const chunks: EmojiCatalogEntry[][] = [];- let current: EmojiCatalogEntry[] = [];- let size = 2;+ for (const hit of hits) {+ if (hit.score < MATCH_SCORE_FLOOR) continue;- for (const entry of catalog) {- const question = emojiQuestion(entry);- const entrySize =- JSON.stringify(entry.id).length + JSON.stringify(question).length + 1;+ const emoji = hit.metadata?.emoji;+ const label = hit.metadata?.label?.trim();- if (current.length > 0 && size + entrySize > MAX_QUESTIONS_JSON_CHARS) {- chunks.push(current);- current = [];- size = 2;- }+ if (!emoji || !label) continue;- current.push(entry);- size += entrySize;- }+ matches.push({ emoji, label });- if (current.length > 0) {- chunks.push(current);+ if (matches.length >= MATCH_LIMIT) break;}- return chunks;-}--function getEmojiCatalog() {- catalogPromise ??= fetch(EMOJIBASE_DATA_URL)- .then(async (response) => {- if (!response.ok) {- throw new Error("Failed to load emoji data");- }-- const data: unknown = await response.json();-- if (!Array.isArray(data)) {- throw new Error("Failed to load emoji data");- }-- const catalog: EmojiCatalogEntry[] = [];- const seen = new Set<string>();-- for (const item of data) {- if (!item || typeof item !== "object") continue;-- const record = item as Record<string, unknown>;- const emoji = typeof record.emoji === "string" ? record.emoji : null;- const label =- typeof record.label === "string" ? record.label.trim() : null;- const hexcode =- typeof record.hexcode === "string" ? record.hexcode : null;- const tags = Array.isArray(record.tags)- ? record.tags.filter((tag): tag is string => typeof tag === "string")- : [];-- if (!emoji || !label || !hexcode || seen.has(hexcode)) continue;-- seen.add(hexcode);- catalog.push({ id: hexcode, emoji, label, tags });- }-- return catalog;- })- .catch((error) => {- catalogPromise = null;- throw error;- });-- return catalogPromise;-}--export function warmEmojiCatalog() {- void getEmojiCatalog().catch(() => undefined);+ return matches;}export async function searchEmojis(query: string): Promise<EmojiMatch[]> {@@ -149,59 +42,11 @@ export async function searchEmojis(query: string): Promise<EmojiMatch[]> {return [];}- const cached = searchCache.get(normalized);- if (cached) return cached;-- const pending = searchRequests.get(normalized);- if (pending) return pending;-- const request = getEmojiCatalog()- .then(async (catalog) => {- const probabilitySets = await Promise.all(- chunkEmojiCatalog(catalog).map(async (entries) => {- const result = await evaluate({- model: "typesafe-ai/jev",- state: normalized,- questions: Object.fromEntries(- entries.map((entry) => [entry.id, emojiQuestion(entry)]),- ),- providerOptions: {- gateway: {- zeroDataRetention: true,- },- },- });-- const probabilities: Record<string, number> = {};-- for (const entry of entries) {- const answer = result.answers[entry.id];-- if (answer?.type === "boolean") {- probabilities[entry.id] = answer.probability;- }- }-- return probabilities;- }),- );- const matches = rankEmojiMatches(- catalog,- Object.assign({}, ...probabilitySets),- );-- if (searchCache.size >= SEARCH_CACHE_LIMIT) {- const oldest = searchCache.keys().next().value;- if (oldest) searchCache.delete(oldest);- }-- searchCache.set(normalized, matches);- return matches;- })- .finally(() => {- searchRequests.delete(normalized);- });+ const hits = await getEmojiVectorIndex().query({+ data: normalized,+ topK: MATCH_LIMIT,+ includeMetadata: true,+ });- searchRequests.set(normalized, request);- return request;+ return toEmojiMatches(hits);}diff --git a/apps/web/lib/upstash/vector.ts b/apps/web/lib/upstash/vector.tsindex 8a3ebec27c..1f54cff39b 100644--- a/apps/web/lib/upstash/vector.ts+++ b/apps/web/lib/upstash/vector.ts@@ -4,3 +4,26 @@ export const vectorIndex = new Index({url: process.env.UPSTASH_VECTOR_REST_URL!,token: process.env.UPSTASH_VECTOR_REST_TOKEN!,});++type EmojiVectorMetadata = {+ emoji: string;+ label: string;+};++let emojiVectorIndex: Index<EmojiVectorMetadata> | null = null;++export function getEmojiVectorIndex() {+ if (emojiVectorIndex) return emojiVectorIndex;++ const url = process.env.UPSTASH_VECTOR_EMOJI_REST_URL;+ const token = process.env.UPSTASH_VECTOR_EMOJI_REST_TOKEN;++ if (!url || !token) {+ throw new Error(+ "Set UPSTASH_VECTOR_EMOJI_REST_URL and UPSTASH_VECTOR_EMOJI_REST_TOKEN",+ );+ }++ emojiVectorIndex = new Index<EmojiVectorMetadata>({ url, token });+ return emojiVectorIndex;+}diff --git a/apps/web/scripts/dev/upsert-emoji-embeddings.ts b/apps/web/scripts/dev/upsert-emoji-embeddings.tsnew file mode 100644index 0000000000..21151b0bf1--- /dev/null+++ b/apps/web/scripts/dev/upsert-emoji-embeddings.ts@@ -0,0 +1,102 @@+import { toEmojiMatches } from "@/lib/ai/search-emojis";+import { getEmojiVectorIndex } from "@/lib/upstash/vector";+import "dotenv-flow/config";++const EMOJIBASE_DATA_URL =+ "https://cdn.jsdelivr.net/npm/[email protected]/en/data.json";+const UPSERT_BATCH_SIZE = 500;++type EmojiVectorRecord = {+ id: string;+ data: string;+ metadata: {+ emoji: string;+ label: string;+ };+};++async function loadEmojiRecords(): Promise<EmojiVectorRecord[]> {+ const response = await fetch(EMOJIBASE_DATA_URL, {+ headers: { "User-Agent": "Mozilla/5.0" },+ });++ if (!response.ok) {+ throw new Error("Failed to load emoji data");+ }++ const data: unknown = await response.json();++ if (!Array.isArray(data)) {+ throw new Error("Failed to load emoji data");+ }++ const records: EmojiVectorRecord[] = [];+ const seen = new Set<string>();++ for (const item of data) {+ if (!item || typeof item !== "object") continue;++ const record = item as Record<string, unknown>;+ const emoji = typeof record.emoji === "string" ? record.emoji : null;+ const label = typeof record.label === "string" ? record.label.trim() : null;+ const hexcode = typeof record.hexcode === "string" ? record.hexcode : null;++ if (!emoji || !label || !hexcode || seen.has(hexcode)) continue;++ const tags = Array.isArray(record.tags)+ ? record.tags.filter(+ (tag): tag is string =>+ typeof tag === "string" &&+ tag.length > 1 &&+ tag.toLowerCase() !== label.toLowerCase(),+ )+ : [];++ seen.add(hexcode);+ records.push({+ id: hexcode,+ data: tags.length === 0 ? label : `${label} (${tags.join(", ")})`,+ metadata: { emoji, label },+ });+ }++ return records;+}++async function main() {+ if (+ !process.env.UPSTASH_VECTOR_EMOJI_REST_URL ||+ !process.env.UPSTASH_VECTOR_EMOJI_REST_TOKEN+ ) {+ throw new Error(+ "Set UPSTASH_VECTOR_EMOJI_REST_URL and UPSTASH_VECTOR_EMOJI_REST_TOKEN",+ );+ }++ const records = await loadEmojiRecords();++ for (let index = 0; index < records.length; index += UPSERT_BATCH_SIZE) {+ const batch = records.slice(index, index + UPSERT_BATCH_SIZE);+ await getEmojiVectorIndex().upsert(batch);+ console.log(+ `Upserted ${Math.min(index + batch.length, records.length)}/${records.length}`,+ );+ }++ const query = "jurassic park";+ const hits = await getEmojiVectorIndex().query({+ data: query,+ topK: 18,+ includeMetadata: true,+ });++ console.log(`\n${query}`);+ for (const match of toEmojiMatches(hits)) {+ console.log(`${match.emoji} ${match.label}`);+ }+}++main().catch((error) => {+ console.error(error);+ process.exit(1);+});diff --git a/apps/web/tests/misc/search-emojis.test.ts b/apps/web/tests/misc/search-emojis.test.tsdeleted file mode 100644index 4312097a75..0000000000--- a/apps/web/tests/misc/search-emojis.test.ts+++ /dev/null@@ -1,63 +0,0 @@-import {- chunkEmojiCatalog,- rankEmojiMatches,- type EmojiCatalogEntry,-} from "@/lib/ai/search-emojis";-import { describe, expect, it } from "vitest";--function entry(- id: string,- emoji: string,- label = "grinning face",-): EmojiCatalogEntry {- return { id, emoji, label };-}--describe("rankEmojiMatches", () => {- const catalog = [- entry("1", "😀", "grinning face"),- entry("2", "🍕", "pizza"),- entry("3", "🤒", "face with thermometer"),- ];-- it("keeps thematic matches above 0.25, highest probability first", () => {- expect(- rankEmojiMatches(catalog, {- "1": 0.3,- "2": 0.25,- "3": 0.91,- }),- ).toEqual([- { emoji: "🤒", label: "face with thermometer" },- { emoji: "😀", label: "grinning face" },- ]);- });-- it("returns at most 18 matches", () => {- const many = Array.from({ length: 50 }, (_, index) =>- entry(String(index), String.fromCodePoint(0x1f600 + (index % 20))),- );-- const probabilities = Object.fromEntries(- many.map((item, index) => [item.id, 0.7 + index / 1000]),- );-- expect(rankEmojiMatches(many, probabilities)).toHaveLength(18);- });-});--describe("chunkEmojiCatalog", () => {- it("keeps a small catalog in one request", () => {- expect(- chunkEmojiCatalog([entry("1F600", "😀"), entry("1F355", "🍕", "pizza")]),- ).toHaveLength(1);- });-- it("splits a catalog that would exceed Jev's context window", () => {- const label = "a".repeat(60_000);-- expect(- chunkEmojiCatalog([entry("1", "😀", label), entry("2", "🍕", label)]),- ).toHaveLength(2);- });-});