The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(kb): add search ingestion and edge function (#51843)
Ali Waseem20e304a37708
diff --git a/.github/workflows/docs-search-v2-ingest.yml b/.github/workflows/docs-search-v2-ingest.ymlindex 5bb47357a2c..a5b04c2f066 100644--- a/.github/workflows/docs-search-v2-ingest.yml+++ b/.github/workflows/docs-search-v2-ingest.yml@@ -8,6 +8,7 @@ on:- '.github/workflows/docs-search-v2-ingest.yml'- 'apps/docs/content/**/*.mdx'- 'apps/docs/scripts/search_v2/**'+ - 'packages/docs-search/**'workflow_dispatch:concurrency:@@ -51,10 +52,6 @@ jobs:- name: Download dependenciesrun: pnpm install --frozen-lockfile- - name: Fetch federated content- working-directory: ./apps/docs- run: pnpm run build:federated-content-- name: Ingest contentworking-directory: ./apps/docs- run: pnpm run search-v2:ingest+ run: pnpm run ingest:search_v2diff --git a/.github/workflows/docs-tests.yml b/.github/workflows/docs-tests.ymlindex 8acf713169a..8855a6d1903 100644--- a/.github/workflows/docs-tests.yml+++ b/.github/workflows/docs-tests.yml@@ -11,6 +11,7 @@ on:- 'apps/docs/content/_partials/access-control/**'- 'apps/docs/.env.development'- 'apps/docs/package.json'+ - 'packages/docs-search/**'- 'packages/shared-data/package.json'- 'packages/shared-data/scoped-access-token-permissions.ts'- 'e2e/docs/local-smoke/**'diff --git a/.github/workflows/kb-search-ingest.yml b/.github/workflows/kb-search-ingest.ymlnew file mode 100644index 00000000000..b88828ec266--- /dev/null+++ b/.github/workflows/kb-search-ingest.yml@@ -0,0 +1,58 @@+name: 'KB Search Ingest'++on:+ push:+ branches:+ - master+ paths:+ - '.github/workflows/kb-search-ingest.yml'+ - 'apps/kb/scripts/**'+ - 'packages/docs-search/**'+ - 'supabase/migrations/*_kb_search*.sql'+ schedule:+ - cron: '0 0 * * *'+ workflow_dispatch:++concurrency:+ group: ${{ github.workflow }}-prod+ cancel-in-progress: false++permissions:+ contents: read++jobs:+ ingest:+ runs-on: blacksmith-4vcpu-ubuntu-2404++ env:+ KB_GITHUB_APP_ID: ${{ secrets.KB_GITHUB_APP_ID }}+ KB_GITHUB_APP_INSTALLATION_ID: ${{ secrets.KB_GITHUB_APP_INSTALLATION_ID }}+ KB_GITHUB_APP_PRIVATE_KEY: ${{ secrets.KB_GITHUB_APP_PRIVATE_KEY }}+ SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }}+ SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }}++ steps:+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0+ with:+ persist-credentials: false+ sparse-checkout: |+ apps/kb+ packages+ patches++ - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9+ name: Install pnpm+ with:+ run_install: false++ - name: Use Node.js+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0+ with:+ node-version-file: '.nvmrc'+ cache: 'pnpm'++ - name: Install dependencies+ run: pnpm install --frozen-lockfile++ - name: Ingest troubleshooting articles+ run: pnpm run --filter kb search:ingestdiff --git a/apps/docs/app/api/graphql/tests/searchDocs.test.ts b/apps/docs/app/api/graphql/tests/searchDocs.test.tsindex 92b9012a862..ba0aaa2707f 100644--- a/apps/docs/app/api/graphql/tests/searchDocs.test.ts+++ b/apps/docs/app/api/graphql/tests/searchDocs.test.ts@@ -59,10 +59,13 @@ vi.mock('~/lib/supabase', () => ({describe('/api/graphql searchDocs', () => {beforeAll(() => {vi.spyOn(console, 'error').mockImplementation(() => {})+ // KB search is a network call; return no KB results+ vi.stubGlobal('fetch', () => Promise.resolve(Response.json([])))})afterAll(() => {vi.restoreAllMocks()+ vi.unstubAllGlobals()vi.doUnmock('~/lib/openAi')vi.doUnmock('~/lib/supabase')})diff --git a/apps/docs/package.json b/apps/docs/package.jsonindex 114063649c1..d4de506c11b 100644--- a/apps/docs/package.json+++ b/apps/docs/package.json@@ -37,11 +37,11 @@"predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples","preembeddings": "pnpm run codegen:references","preinstall": "npx only-allow pnpm",- "presearch-v2:ingest": "pnpm run build:guides-markdown",+ "preingest:search_v2": "pnpm run build:federated-content && pnpm run build:guides-markdown","presync": "pnpm run codegen:graphql","pretest": "pnpm run codegen:examples","pretypecheck": "pnpm run codegen:graphql && next typegen",- "search-v2:ingest": "tsx scripts/search_v2/ingest.ts",+ "ingest:search_v2": "tsx scripts/search_v2/ingest.ts","start": "next start","sync": "tsx --conditions=react-server ./resources/rootSync.ts","test": "pnpm supabase start && pnpm run test:local && pnpm supabase stop",@@ -70,6 +70,7 @@"beautiful-mermaid": "^1.1.3","class-variance-authority": "^0.7.1","common": "workspace:*",+ "docs-search": "workspace:*","common-tags": "^1.8.2","config": "workspace:*","dev-tools": "workspace:*",diff --git a/apps/docs/resources/globalSearch/globalSearchModel.ts b/apps/docs/resources/globalSearch/globalSearchModel.tsindex 0d6b4881f61..66fe5bf67d1 100644--- a/apps/docs/resources/globalSearch/globalSearchModel.ts+++ b/apps/docs/resources/globalSearch/globalSearchModel.ts@@ -15,6 +15,8 @@ import { ReferenceSDKFunctionModel, SDKLanguageValues } from '../reference/referimport { TroubleshootingModel } from '../troubleshooting/troubleshootingModel'import { SearchResultInterface } from './globalSearchInterface'+const KB_RESULTS = 5+type SearchFunction = 'search_content' | 'search_content_nimbus'type SearchHybridFunction = 'search_content_hybrid' | 'search_content_hybrid_nimbus'@@ -57,6 +59,9 @@ export abstract class SearchResultModel {): Promise<Result<SearchResultModel[], ApiErrorGeneric>> {const query = args.query.trim()const includeFullContent = requestedFields.includes('content')+ const limit = args.limit ?? 30+ const kbLimit = Math.min(Math.max(limit - KB_RESULTS, 0), KB_RESULTS)+ const kbSearch = searchKb(query, kbLimit)const embeddingResult = await openAI().createContentEmbedding(query)const useAltSearchIndex = !isFeatureEnabled('search:fullIndex')@@ -64,13 +69,13 @@ export abstract class SearchResultModel {? 'search_content_hybrid_nimbus': 'search_content_hybrid'- return embeddingResult.flatMapAsync(async ({ embedding }) => {+ const docsResult = await embeddingResult.flatMapAsync(async ({ embedding }) => {const matchResult = new Result(await supabase().rpc(searchFunction, {query_text: query,query_embedding: embedding,include_full_content: includeFullContent,- max_result: args.limit ?? 30,+ max_result: limit - kbLimit,})).map((matches) =>@@ -82,6 +87,30 @@ export abstract class SearchResultModel {return matchResult})++ const kbResults = await kbSearch+ return docsResult.map((items) => [...items, ...kbResults])+ }+}++/** Fails open to no results so docs search keeps working if KB is slow or down. */+async function searchKb(query: string, limit: number): Promise<SearchResultInterface[]> {+ if (limit <= 0) return []++ try {+ const params = new URLSearchParams({ q: query, limit: String(limit), content: 'true' })+ const response = await fetch(`https://supabase.com/kb/api/search?${params}`, {+ signal: AbortSignal.timeout(3000),+ })+ if (!response.ok) throw new Error(`KB search responded with ${response.status}`)++ return (await response.json()).map(({ title, href, content }: Record<string, string>) => {+ const Model = href.includes('/kb/troubleshooting/') ? TroubleshootingModel : GuideModel+ return new Model({ title, href, content: content ?? undefined })+ })+ } catch (error) {+ console.warn('Error searching KB:', error)+ return []}}diff --git a/apps/docs/scripts/search_v2/ingest.ts b/apps/docs/scripts/search_v2/ingest.tsindex 89767484b7e..6c94a83b961 100644--- a/apps/docs/scripts/search_v2/ingest.ts+++ b/apps/docs/scripts/search_v2/ingest.ts@@ -1,112 +1,20 @@import '../utils/dotenv.js'-import { readFile } from 'node:fs/promises'import path from 'node:path'-import { fileURLToPath } from 'node:url'-import fg from 'fast-glob'+import { ingestMarkdown } from 'docs-search/ingest'-import { parsePage } from './markdown.js'-import { filePathToSlug } from './routes.js'import { createSupabaseClient, TABLE_NAME } from './supabase.js'-// ----------------------------------------------------------------------------// Configuration-// -----------------------------------------------------------------------------/** Folder that holds the markdown files. Searched recursively. */-export const CONTENT_ROOT = path.resolve(process.cwd(), 'public/markdown')--/** Glob patterns (relative to CONTENT_ROOT) for the files to ingest. */-const FILE_PATTERNS = ['**/*.md', '**/*.mdx']--/** Glob patterns (relative to CONTENT_ROOT) for files/folders to skip. */-const IGNORED_PATTERNS = ['reference/**', 'guides/troubleshooting/**', '_partials/**']--/** How many rows to send to Supabase per insert request. */-const INSERT_BATCH_SIZE = 200--// -----------------------------------------------------------------------------interface SectionRow {- slug: string- file_path: string- page_title: string- heading: string- heading_level: number- heading_path: string[]- content: string- excerpt: string-}--/** Build the DB rows for a single markdown file. Pure: no I/O. */-export function buildRows(markdown: string, filePath: string, contentRoot: string): SectionRow[] {- const slug = filePathToSlug(filePath, contentRoot)- const relativePath = path.relative(contentRoot, filePath).split(path.sep).join('/')- const page = parsePage(markdown)- const pageTitle = page.title || path.basename(slug) || 'Untitled'-- return page.sections.map((section) => ({- slug,- file_path: relativePath,- page_title: pageTitle,- heading: section.heading,- heading_level: section.level,- heading_path: section.headingPath,- content: section.content,- excerpt: page.excerpt,- }))-}--function chunk<T>(items: T[], size: number): T[][] {- const out: T[][] = []- for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))- return out-}-async function main() {- const supabase = createSupabaseClient()-- const files = await fg(FILE_PATTERNS, {- cwd: CONTENT_ROOT,- absolute: true,- onlyFiles: true,- ignore: IGNORED_PATTERNS,+ await ingestMarkdown({+ supabase: createSupabaseClient(),+ table: TABLE_NAME,+ contentRoot: path.resolve(process.cwd(), 'public/markdown'),+ ignore: ['reference/**', 'guides/troubleshooting/**', '_partials/**'],})- if (files.length === 0) {- console.warn(`No markdown files found under ${CONTENT_ROOT}`)- return- }- console.log(`Found ${files.length} markdown file(s) under ${CONTENT_ROOT}`)-- let totalRows = 0-- for (const file of files.sort()) {- const markdown = await readFile(file, 'utf8')- const rows = buildRows(markdown, file, CONTENT_ROOT)- const slug = rows[0]?.slug ?? filePathToSlug(file, CONTENT_ROOT)-- // Replace everything previously ingested for this page.- const { error: deleteError } = await supabase.from(TABLE_NAME).delete().eq('slug', slug)- if (deleteError) throw new Error(`Failed to clear "${slug}": ${deleteError.message}`)-- for (const batch of chunk(rows, INSERT_BATCH_SIZE)) {- const { error: insertError } = await supabase.from(TABLE_NAME).insert(batch)- if (insertError) throw new Error(`Failed to insert "${slug}": ${insertError.message}`)- }-- totalRows += rows.length- console.log(` ✓ ${slug || '(root)'} (${rows.length} section${rows.length === 1 ? '' : 's'})`)- }-- console.log(`Done. Ingested ${totalRows} section(s) from ${files.length} file(s).`)}-// Only run when executed directly (so buildRows can be imported by tests).-const isDirectRun =- !!process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)-if (isDirectRun) {- main().catch((err) => {- console.error(err instanceof Error ? err.message : err)- process.exit(1)- })-}+main().catch((err) => {+ console.error(err instanceof Error ? err.message : err)+ process.exit(1)+})diff --git a/apps/docs/scripts/search_v2/sql/setup.sql b/apps/docs/scripts/search_v2/sql/setup.sqlindex 203f1212183..09ef0b1acf5 100644--- a/apps/docs/scripts/search_v2/sql/setup.sql+++ b/apps/docs/scripts/search_v2/sql/setup.sql@@ -1,5 +1,5 @@--- Run this in the Supabase SQL editor before `pnpm search-v2:ingest`.+-- Run this in the Supabase SQL editor before `pnpm ingest:search_v2`.-- Safe to re-run: it uses IF NOT EXISTS / CREATE OR REPLACE.-- One row per markdown *section* (a heading + the text below it, up to the next heading).diff --git a/apps/kb/.env.example b/apps/kb/.env.exampleindex eed0248c52f..49d09643de9 100644--- a/apps/kb/.env.example+++ b/apps/kb/.env.example@@ -15,6 +15,7 @@ GH_TOKEN=# Supabase project backing troubleshooting_entries (same project apps/www and# apps/studio use — supabase/config.toml's `[remotes.prod]`). Everything in kb# uses the service role key: the Astro build reads with it, the sync scripts-# also write. Never expose it to client code (kb is static, build-time only).+# also write, and api/search.ts queries kb_search at runtime. Never+# expose it to client code.SUPABASE_URL=SUPABASE_SERVICE_ROLE_KEY=diff --git a/apps/kb/api/search.ts b/apps/kb/api/search.tsnew file mode 100644index 00000000000..0c6a57de7f8--- /dev/null+++ b/apps/kb/api/search.ts@@ -0,0 +1,75 @@+import { createClient, type SupabaseClient } from '@supabase/supabase-js'++import { corsHeaders } from '../src/lib/search/cors'+import type { Database, KbSearchResult } from '../src/lib/search/types'++let client: SupabaseClient<Database> | undefined+function searchClient() {+ client ??= createClient<Database>(+ process.env.SUPABASE_URL!,+ process.env.SUPABASE_SERVICE_ROLE_KEY!,+ { auth: { autoRefreshToken: false, persistSession: false, detectSessionInUrl: false } }+ )+ return client+}++/** Fetches the markdown export of a page (`<href>.md`); null if it can't be loaded. */+async function fetchContent(href: string): Promise<string | null> {+ try {+ const response = await fetch(`${href}.md`)+ if (!response.ok) {+ console.warn(`Failed to fetch kb content from ${href}.md: ${response.status}`)+ return null+ }+ return await response.text()+ } catch (error) {+ console.warn(`Failed to fetch kb content from ${href}.md:`, error)+ return null+ }+}++export function OPTIONS() {+ return new Response(null, { headers: corsHeaders })+}++export async function GET(request: Request) {+ const { searchParams } = new URL(request.url)+ const query = searchParams.get('q')?.trim()+ const limit = Number(searchParams.get('limit')) || 10+ const includeContent = searchParams.get('content') === 'true'+ if (!query)+ return Response.json({ error: 'Missing q parameter' }, { status: 400, headers: corsHeaders })++ const { data, error } = await searchClient().rpc('search_kb', {+ query_text: query,+ match_limit: limit,+ })+ if (error) {+ console.error('Error running kb search:', error)+ return Response.json(+ { error: 'There was an error processing your request' },+ { status: 500, headers: corsHeaders }+ )+ }++ let results: KbSearchResult[] = data.map((row) => ({+ title: row.page_title,+ href: `https://supabase.com/${row.slug.toLowerCase()}`,+ heading: row.heading,+ excerpt: row.excerpt,+ score: row.score,+ }))++ if (includeContent) {+ results = await Promise.all(+ results.map(async (result) => ({ ...result, content: await fetchContent(result.href) }))+ )+ }++ return Response.json(results, {+ headers: {+ ...corsHeaders,+ 'Cache-Control': 'public, s-maxage=3600, stale-while-revalidate=300',+ },+ })+}diff --git a/apps/kb/package.json b/apps/kb/package.jsonindex ccc92ca4bb9..9e0135c48bd 100644--- a/apps/kb/package.json+++ b/apps/kb/package.json@@ -11,6 +11,8 @@"build": "astro build","fetch:federated-content": "tsx ./scripts/federated-content/fetch-federated-content.ts","generate:markdown": "tsx ./scripts/generate-markdown.mjs",+ "presearch:ingest": "pnpm run fetch:federated-content && pnpm run generate:markdown",+ "search:ingest": "tsx ./scripts/search/ingest.ts","sync:legacy-slugs": "tsx ./scripts/federated-content/backfill-legacy-slugs.ts","sync:troubleshooting-entries": "tsx ./scripts/federated-content/sync-troubleshooting-entries.ts","sync:troubleshooting-updates": "tsx ./scripts/federated-content/sync-troubleshooting-updates.ts",@@ -29,6 +31,7 @@"@supabase/supabase-js": "catalog:","astro": "^7.2.6","common": "workspace:*",+ "docs-search": "workspace:*","gray-matter": "^4.0.3","lucide-react": "catalog:","react": "catalog:",diff --git a/apps/kb/scripts/search/ingest.ts b/apps/kb/scripts/search/ingest.tsnew file mode 100644index 00000000000..79bb958d2e8--- /dev/null+++ b/apps/kb/scripts/search/ingest.ts@@ -0,0 +1,26 @@+import '../utils/dotenv.js'++import path from 'node:path'+import { createClient } from '@supabase/supabase-js'+import { ingestMarkdown } from 'docs-search/ingest'++import { TABLE_NAME } from './supabase.js'++const { SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY } = process.env+if (!SUPABASE_URL || !SUPABASE_SERVICE_ROLE_KEY) {+ console.error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY. Set them in .env.local.')+ process.exit(1)+}++ingestMarkdown({+ supabase: createClient(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, {+ auth: { autoRefreshToken: false, persistSession: false },+ }),+ table: TABLE_NAME,+ contentRoot: path.resolve(process.cwd(), 'public/markdown'),+ include: ['**/*.md'],+ slugPrefix: 'kb/',+}).catch((err) => {+ console.error(err instanceof Error ? err.message : err)+ process.exit(1)+})diff --git a/apps/kb/scripts/search/supabase.ts b/apps/kb/scripts/search/supabase.tsnew file mode 100644index 00000000000..543874c0adb--- /dev/null+++ b/apps/kb/scripts/search/supabase.ts@@ -0,0 +1 @@+export const TABLE_NAME = 'kb_search'diff --git a/apps/kb/src/lib/search/cors.ts b/apps/kb/src/lib/search/cors.tsnew file mode 100644index 00000000000..c7243dfacd1--- /dev/null+++ b/apps/kb/src/lib/search/cors.ts@@ -0,0 +1,5 @@+export const corsHeaders = {+ 'Access-Control-Allow-Origin': '*',+ 'Access-Control-Allow-Methods': 'GET, OPTIONS',+ 'Access-Control-Allow-Headers': 'content-type',+}diff --git a/apps/kb/src/lib/search/types.ts b/apps/kb/src/lib/search/types.tsnew file mode 100644index 00000000000..07c11e69d34--- /dev/null+++ b/apps/kb/src/lib/search/types.ts@@ -0,0 +1,33 @@+export interface KbSearchRow {+ slug: string+ page_title: string+ heading: string+ heading_level: number+ heading_path: string[]+ excerpt: string+ score: number+}++export interface KbSearchResult {+ title: string+ href: string+ heading: string+ excerpt: string+ score: number+ content?: string | null+}++export type Database = {+ public: {+ Tables: Record<string, never>+ Views: Record<string, never>+ Functions: {+ search_kb: {+ Args: { query_text: string; match_limit?: number }+ Returns: KbSearchRow[]+ }+ }+ Enums: Record<string, never>+ CompositeTypes: Record<string, never>+ }+}diff --git a/package.json b/package.jsonindex 1a77eeabed3..7d806865f65 100644--- a/package.json+++ b/package.json@@ -27,7 +27,7 @@"test:case-hazards": "node scripts/check-case-hazards.mjs","test:prettier": "SORT_IMPORTS=false prettier --config prettier.config.mjs --cache --check '{apps,packages,blocks,examples,i18n}/**/*.{js,jsx,ts,tsx,css,md,mdx,json}'","format": "SORT_IMPORTS=false prettier --config prettier.config.mjs --cache --write '{apps,packages,blocks,examples,i18n}/**/*.{js,jsx,ts,tsx,css,md,mdx,json}'",- "test:docs": "turbo run test --filter=docs",+ "test:docs": "turbo run test --filter=docs --filter=docs-search","test:kb": "turbo run test --filter=kb","test:ui": "turbo run test --filter=ui","test:ui-patterns": "turbo run test --filter=ui-patterns",diff --git a/packages/docs-search/package.json b/packages/docs-search/package.jsonnew file mode 100644index 00000000000..fcd7fe66368--- /dev/null+++ b/packages/docs-search/package.json@@ -0,0 +1,31 @@+{+ "name": "docs-search",+ "version": "0.0.0",+ "license": "MIT",+ "scripts": {+ "preinstall": "npx only-allow pnpm",+ "clean": "rimraf .turbo tsconfig.tsbuildinfo",+ "typecheck": "tsc --noEmit",+ "test": "vitest --run"+ },+ "dependencies": {+ "fast-glob": "^3.3.3",+ "mdast-util-to-string": "^3.1.1",+ "remark-frontmatter": "^5.0.0",+ "remark-gfm": "^4.0.0",+ "remark-parse": "^11.0.0",+ "unified": "^11.0.5"+ },+ "devDependencies": {+ "@types/mdast": "^3.0.0",+ "@types/node": "catalog:",+ "@typescript/native": "catalog:",+ "tsconfig": "workspace:*",+ "typescript": "catalog:",+ "vitest": "catalog:",+ "@supabase/supabase-js": "catalog:"+ },+ "exports": {+ "./ingest": "./src/ingest.ts"+ }+}diff --git a/packages/docs-search/src/ingest.ts b/packages/docs-search/src/ingest.tsnew file mode 100644index 00000000000..00a398270e9--- /dev/null+++ b/packages/docs-search/src/ingest.ts@@ -0,0 +1,52 @@+import { readFile } from 'node:fs/promises'+import fg from 'fast-glob'++import { filePathToSlug } from './routes.js'+import { buildRows } from './rows.js'+import type { IngestOptions } from './types.js'++const INSERT_BATCH_SIZE = 200++function chunk<T>(items: T[], size: number): T[][] {+ const out: T[][] = []+ for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))+ return out+}++export async function ingestMarkdown({+ supabase,+ contentRoot,+ include = ['**/*.md', '**/*.mdx'],+ ignore = [],+ table,+ slugPrefix = '',+}: IngestOptions): Promise<number> {+ const files = await fg(include, { cwd: contentRoot, absolute: true, onlyFiles: true, ignore })+ if (files.length === 0) {+ console.warn(`No markdown files found under ${contentRoot}`)+ return 0+ }+ console.log(`Found ${files.length} markdown file(s) under ${contentRoot}`)++ let totalRows = 0++ for (const file of files.sort()) {+ const markdown = await readFile(file, 'utf8')+ const rows = buildRows(markdown, file, contentRoot, slugPrefix)+ const slug = rows[0]?.slug ?? slugPrefix + filePathToSlug(file, contentRoot)++ const { error: deleteError } = await supabase.from(table).delete().eq('slug', slug)+ if (deleteError) throw new Error(`Failed to clear "${slug}": ${deleteError.message}`)++ for (const batch of chunk(rows, INSERT_BATCH_SIZE)) {+ const { error: insertError } = await supabase.from(table).insert(batch)+ if (insertError) throw new Error(`Failed to insert "${slug}": ${insertError.message}`)+ }++ totalRows += rows.length+ console.log(` ✓ ${slug || '(root)'} (${rows.length} section${rows.length === 1 ? '' : 's'})`)+ }++ console.log(`Done. Ingested ${totalRows} section(s) from ${files.length} file(s).`)+ return totalRows+}diff --git a/apps/docs/scripts/search_v2/markdown.test.ts b/packages/docs-search/src/markdown.test.tssimilarity index 100%rename from apps/docs/scripts/search_v2/markdown.test.tsrename to packages/docs-search/src/markdown.test.tsdiff --git a/apps/docs/scripts/search_v2/markdown.ts b/packages/docs-search/src/markdown.tssimilarity index 71%rename from apps/docs/scripts/search_v2/markdown.tsrename to packages/docs-search/src/markdown.tsindex 7fb4c26d334..a247efa2186 100644--- a/apps/docs/scripts/search_v2/markdown.ts+++ b/packages/docs-search/src/markdown.ts@@ -4,39 +4,20 @@ import remarkGfm from 'remark-gfm'import remarkParse from 'remark-parse'import { unified } from 'unified'+import type { ParsedPage, Section } from './types.js'+const processor = unified().use(remarkParse).use(remarkFrontmatter, ['yaml']).use(remarkGfm)/**- * Types are derived from the processor's own output rather than imported from- * `mdast` directly: this app pins `@types/mdast` v3 for its wider MDX pipeline,- * while `remark-parse` v11 resolves its own (structurally incompatible) v4- * types for its nested `@types/mdast` dependency.+ * Types are derived from the processor's output rather than imported from `mdast`:+ * the docs app pins `@types/mdast` v3, while `remark-parse` v11 resolves its own+ * (structurally incompatible) v4 types.*/export type Root = ReturnType<typeof processor.parse>-/** Any child of the root node (paragraph, heading, list, table, ...). */type Content = Root['children'][number]type Heading = Extract<Content, { type: 'heading' }>-/** Union of every node type that can appear in the tree (root or content). */type Nodes = Root | Content-export interface Section {- /** Text of this section's heading ('' for text that appears before any heading). */- heading: string- /** 1..6, or 0 for text before any heading. */- level: number- /** Headings from the H1 down to this section, e.g. ['Users', 'Permanent and anonymous users']. */- headingPath: string[]- /** Plain text of everything under the heading until the next heading. */- content: string-}--export interface ParsedPage {- title: string- excerpt: string- sections: Section[]-}--/** Parse markdown into an mdast tree (GFM tables/lists + YAML frontmatter supported). */export function parseMarkdownAst(markdown: string): Root {return processor.parse(markdown)}@@ -59,7 +40,6 @@ export function nodeToText(node: Nodes): string {}}-/** The first H1 of the document, or '' if there is none. */export function extractTitle(tree: Root): string {const h1 = tree.children.find((node): node is Heading => node.type === 'heading' && node.depth === 1@@ -73,7 +53,6 @@ export function extractExcerpt(tree: Root): string {return paragraph ? nodeToText(paragraph) : ''}-/** Nodes that carry no searchable text. */function isIgnored(node: Content): boolean {return (node.type === 'yaml' ||@@ -118,7 +97,6 @@ export function extractSections(tree: Root): Section[] {continue}- // Body text before the first heading gets a headless section.if (!current) current = { heading: '', level: 0, headingPath: [], content: '' }const text = nodeToText(node)if (text) body.push(text)@@ -128,7 +106,6 @@ export function extractSections(tree: Root): Section[] {return sections}-/** Everything the ingester needs from one markdown file. */export function parsePage(markdown: string): ParsedPage {const tree = parseMarkdownAst(markdown)return {diff --git a/apps/docs/scripts/search_v2/routes.test.ts b/packages/docs-search/src/routes.test.tssimilarity index 100%rename from apps/docs/scripts/search_v2/routes.test.tsrename to packages/docs-search/src/routes.test.tsdiff --git a/apps/docs/scripts/search_v2/routes.ts b/packages/docs-search/src/routes.tssimilarity index 100%rename from apps/docs/scripts/search_v2/routes.tsrename to packages/docs-search/src/routes.tsdiff --git a/packages/docs-search/src/rows.ts b/packages/docs-search/src/rows.tsnew file mode 100644index 00000000000..419c230df1f--- /dev/null+++ b/packages/docs-search/src/rows.ts@@ -0,0 +1,28 @@+import path from 'node:path'++import { parsePage } from './markdown.js'+import { filePathToSlug } from './routes.js'+import type { SectionRow } from './types.js'++export function buildRows(+ markdown: string,+ filePath: string,+ contentRoot: string,+ slugPrefix = ''+): SectionRow[] {+ const slug = slugPrefix + filePathToSlug(filePath, contentRoot)+ const relativePath = path.relative(contentRoot, filePath).split(path.sep).join('/')+ const page = parsePage(markdown)+ const pageTitle = page.title || path.basename(slug) || 'Untitled'++ return page.sections.map((section) => ({+ slug,+ file_path: relativePath,+ page_title: pageTitle,+ heading: section.heading,+ heading_level: section.level,+ heading_path: section.headingPath,+ content: section.content,+ excerpt: page.excerpt,+ }))+}diff --git a/packages/docs-search/src/types.ts b/packages/docs-search/src/types.tsnew file mode 100644index 00000000000..66ade5c875b--- /dev/null+++ b/packages/docs-search/src/types.ts@@ -0,0 +1,34 @@+import type { SupabaseClient } from '@supabase/supabase-js'++export interface Section {+ heading: string+ level: number+ headingPath: string[]+ content: string+}++export interface ParsedPage {+ title: string+ excerpt: string+ sections: Section[]+}++export interface SectionRow {+ slug: string+ file_path: string+ page_title: string+ heading: string+ heading_level: number+ heading_path: string[]+ content: string+ excerpt: string+}++export interface IngestOptions {+ supabase: SupabaseClient+ contentRoot: string+ include?: string[]+ ignore?: string[]+ table: string+ slugPrefix?: string+}diff --git a/packages/docs-search/tsconfig.json b/packages/docs-search/tsconfig.jsonnew file mode 100644index 00000000000..2425077f19f--- /dev/null+++ b/packages/docs-search/tsconfig.json@@ -0,0 +1,9 @@+{+ "extends": "tsconfig/react-library.json",+ "include": ["."],+ "exclude": ["dist", "build", "node_modules"],+ "compilerOptions": {+ "declaration": false,+ "declarationMap": false+ }+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 06b006b48d8..07c70f0b5b6 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -422,6 +422,9 @@ importers:dev-tools:specifier: workspace:*version: link:../../packages/dev-tools+ docs-search:+ specifier: workspace:*+ version: link:../../packages/docs-searcheslint-config-supabase:specifier: workspace:*version: link:../../packages/eslint-config-supabase@@ -774,6 +777,9 @@ importers:common:specifier: workspace:*version: link:../../packages/common+ docs-search:+ specifier: workspace:*+ version: link:../../packages/docs-searchgray-matter:specifier: ^4.0.3version: 4.0.3@@ -2560,6 +2566,49 @@ importers:specifier: 'catalog:'version: 5.0.0(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])([email protected])([email protected](@noble/[email protected])([email protected]))([email protected](@types/[email protected])([email protected]))([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))+ packages/docs-search:+ dependencies:+ fast-glob:+ specifier: ^3.3.3+ version: 3.3.3+ mdast-util-to-string:+ specifier: ^3.1.1+ version: 3.2.0+ remark-frontmatter:+ specifier: ^5.0.0+ version: 5.0.0([email protected])+ remark-gfm:+ specifier: ^4.0.0+ version: 4.0.1([email protected])+ remark-parse:+ specifier: ^11.0.0+ version: 11.0.0([email protected])+ unified:+ specifier: ^11.0.5+ version: 11.0.5+ devDependencies:+ '@supabase/supabase-js':+ specifier: 'catalog:'+ version: 2.117.3(@opentelemetry/[email protected])+ '@types/mdast':+ specifier: ^3.0.0+ version: 3.0.15+ '@types/node':+ specifier: 'catalog:'+ version: 22.13.14+ '@typescript/native':+ specifier: 'catalog:'+ version: [email protected]+ tsconfig:+ specifier: workspace:*+ version: link:../tsconfig+ typescript:+ specifier: 'catalog:'+ version: 6.0.2+ vitest:+ specifier: 'catalog:'+ version: 5.0.0(@opentelemetry/[email protected])(@types/[email protected])(@vitest/[email protected])(@vitest/[email protected])([email protected])([email protected](@noble/[email protected])([email protected]))([email protected](@types/[email protected])([email protected]))([email protected](@types/[email protected])([email protected])([email protected])([email protected])([email protected])([email protected])([email protected]))+packages/eslint-config-supabase:devDependencies:'@shadcn/lint':diff --git a/supabase/migrations/20261016120000_create_kb_search.sql b/supabase/migrations/20261016120000_create_kb_search.sqlnew file mode 100644index 00000000000..670d8ac0351--- /dev/null+++ b/supabase/migrations/20261016120000_create_kb_search.sql@@ -0,0 +1,88 @@+create table public.kb_search (+ id bigint generated always as identity primary key,+ slug text not null,+ file_path text not null,+ page_title text not null default '',+ heading text not null default '',+ heading_level smallint not null default 0,+ heading_path text[] not null default '{}',+ content text not null default '',+ excerpt text not null default '',+ updated_at timestamptz not null default now(),+ fts tsvector generated always as (+ setweight(to_tsvector('english', coalesce(page_title, '')), 'A')+ || setweight(+ to_tsvector('english', coalesce(heading, '')),+ (case heading_level when 1 then 'A' when 2 then 'B' when 3 then 'C' else 'D' end)::"char"+ )+ || setweight(to_tsvector('english', coalesce(content, '')), 'D')+ ) stored+);++create index kb_search_fts_idx on public.kb_search using gin (fts);+create index kb_search_slug_idx on public.kb_search (slug);++alter table public.kb_search enable row level security;++grant select, insert, delete on public.kb_search to service_role;++create function public.search_kb(query_text text, match_limit int default 10)+returns table (+ slug text,+ page_title text,+ heading text,+ heading_level smallint,+ heading_path text[],+ excerpt text,+ score int+)+language sql+stable+set search_path = ''+as $$+ with terms as (+ select distinct+ to_tsquery('simple', quote_literal(lexeme)) as anywhere,+ to_tsquery('simple', quote_literal(lexeme) || ':AB') as in_title+ from unnest(tsvector_to_array(to_tsvector('english', query_text))) as lexeme+ ),+ weighted_terms as (+ select+ t.anywhere,+ t.in_title,+ greatest(0, ln(+ (select count(*) from public.kb_search)::float+ / (1 + (select count(*) from public.kb_search s where s.fts @@ t.anywhere))+ )) as idf+ from terms t+ ),+ ranked as (+ select+ s.slug,+ s.page_title,+ s.heading,+ s.heading_level,+ s.heading_path,+ s.excerpt,+ round(+ 1000 * sum(w.idf * case when s.fts @@ w.in_title then 2 else 1 end)+ / nullif((select 2 * sum(idf) from weighted_terms), 0)+ )::int as score+ from public.kb_search s+ join weighted_terms w on s.fts @@ w.anywhere+ group by s.id+ ),+ best_per_page as (+ select distinct on (r.slug) r.*+ from ranked r+ where r.score >= 300+ order by r.slug, r.score desc, r.heading_level asc+ )+ select b.slug, b.page_title, b.heading, b.heading_level, b.heading_path, b.excerpt, b.score+ from best_per_page b+ order by b.score desc, b.heading_level asc, b.slug asc+ limit greatest(1, least(coalesce(match_limit, 10), 100));+$$;++revoke execute on function public.search_kb(text, int) from public, anon, authenticated;+grant execute on function public.search_kb(text, int) to service_role;