Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(kb): add search ingestion and edge function (#51843)

Ali Waseem20e304a37708
diff --git a/.github/workflows/docs-search-v2-ingest.yml b/.github/workflows/docs-search-v2-ingest.yml
index 5bb47357a2c..a5b04c2f066 100644
--- a/.github/workflows/docs-search-v2-ingest.yml
+++ b/.github/workflows/docs-search-v2-ingest.yml
@@ -8,6 +8,7 @@ on:
- '.github/workflows/docs-search-v2-ingest.yml'
- 'apps/docs/content/**/*.mdx'
- 'apps/docs/scripts/search_v2/**'
+ - 'packages/docs-search/**'
workflow_dispatch:
concurrency:
@@ -51,10 +52,6 @@ jobs:
- name: Download dependencies
run: pnpm install --frozen-lockfile
- - name: Fetch federated content
- working-directory: ./apps/docs
- run: pnpm run build:federated-content
-
- name: Ingest content
working-directory: ./apps/docs
- run: pnpm run search-v2:ingest
+ run: pnpm run ingest:search_v2
diff --git a/.github/workflows/docs-tests.yml b/.github/workflows/docs-tests.yml
index 8acf713169a..8855a6d1903 100644
--- a/.github/workflows/docs-tests.yml
+++ b/.github/workflows/docs-tests.yml
@@ -11,6 +11,7 @@ on:
- 'apps/docs/content/_partials/access-control/**'
- 'apps/docs/.env.development'
- 'apps/docs/package.json'
+ - 'packages/docs-search/**'
- 'packages/shared-data/package.json'
- 'packages/shared-data/scoped-access-token-permissions.ts'
- 'e2e/docs/local-smoke/**'
diff --git a/.github/workflows/kb-search-ingest.yml b/.github/workflows/kb-search-ingest.yml
new file mode 100644
index 00000000000..b88828ec266
--- /dev/null
+++ b/.github/workflows/kb-search-ingest.yml
@@ -0,0 +1,58 @@
+name: 'KB Search Ingest'
+
+on:
+ push:
+ branches:
+ - master
+ paths:
+ - '.github/workflows/kb-search-ingest.yml'
+ - 'apps/kb/scripts/**'
+ - 'packages/docs-search/**'
+ - 'supabase/migrations/*_kb_search*.sql'
+ schedule:
+ - cron: '0 0 * * *'
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-prod
+ cancel-in-progress: false
+
+permissions:
+ contents: read
+
+jobs:
+ ingest:
+ runs-on: blacksmith-4vcpu-ubuntu-2404
+
+ env:
+ KB_GITHUB_APP_ID: ${{ secrets.KB_GITHUB_APP_ID }}
+ KB_GITHUB_APP_INSTALLATION_ID: ${{ secrets.KB_GITHUB_APP_INSTALLATION_ID }}
+ KB_GITHUB_APP_PRIVATE_KEY: ${{ secrets.KB_GITHUB_APP_PRIVATE_KEY }}
+ SUPABASE_URL: ${{ secrets.SEARCH_SUPABASE_URL }}
+ SUPABASE_SERVICE_ROLE_KEY: ${{ secrets.SEARCH_SUPABASE_SERVICE_ROLE_KEY }}
+
+ steps:
+ - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+ with:
+ persist-credentials: false
+ sparse-checkout: |
+ apps/kb
+ packages
+ patches
+
+ - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9
+ name: Install pnpm
+ with:
+ run_install: false
+
+ - name: Use Node.js
+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+ with:
+ node-version-file: '.nvmrc'
+ cache: 'pnpm'
+
+ - name: Install dependencies
+ run: pnpm install --frozen-lockfile
+
+ - name: Ingest troubleshooting articles
+ run: pnpm run --filter kb search:ingest
diff --git a/apps/docs/app/api/graphql/tests/searchDocs.test.ts b/apps/docs/app/api/graphql/tests/searchDocs.test.ts
index 92b9012a862..ba0aaa2707f 100644
--- a/apps/docs/app/api/graphql/tests/searchDocs.test.ts
+++ b/apps/docs/app/api/graphql/tests/searchDocs.test.ts
@@ -59,10 +59,13 @@ vi.mock('~/lib/supabase', () => ({
describe('/api/graphql searchDocs', () => {
beforeAll(() => {
vi.spyOn(console, 'error').mockImplementation(() => {})
+ // KB search is a network call; return no KB results
+ vi.stubGlobal('fetch', () => Promise.resolve(Response.json([])))
})
afterAll(() => {
vi.restoreAllMocks()
+ vi.unstubAllGlobals()
vi.doUnmock('~/lib/openAi')
vi.doUnmock('~/lib/supabase')
})
diff --git a/apps/docs/package.json b/apps/docs/package.json
index 114063649c1..d4de506c11b 100644
--- a/apps/docs/package.json
+++ b/apps/docs/package.json
@@ -37,11 +37,11 @@
"predev": "pnpm run codegen:graphql && pnpm run codegen:references && pnpm run codegen:examples",
"preembeddings": "pnpm run codegen:references",
"preinstall": "npx only-allow pnpm",
- "presearch-v2:ingest": "pnpm run build:guides-markdown",
+ "preingest:search_v2": "pnpm run build:federated-content && pnpm run build:guides-markdown",
"presync": "pnpm run codegen:graphql",
"pretest": "pnpm run codegen:examples",
"pretypecheck": "pnpm run codegen:graphql && next typegen",
- "search-v2:ingest": "tsx scripts/search_v2/ingest.ts",
+ "ingest:search_v2": "tsx scripts/search_v2/ingest.ts",
"start": "next start",
"sync": "tsx --conditions=react-server ./resources/rootSync.ts",
"test": "pnpm supabase start && pnpm run test:local && pnpm supabase stop",
@@ -70,6 +70,7 @@
"beautiful-mermaid": "^1.1.3",
"class-variance-authority": "^0.7.1",
"common": "workspace:*",
+ "docs-search": "workspace:*",
"common-tags": "^1.8.2",
"config": "workspace:*",
"dev-tools": "workspace:*",
diff --git a/apps/docs/resources/globalSearch/globalSearchModel.ts b/apps/docs/resources/globalSearch/globalSearchModel.ts
index 0d6b4881f61..66fe5bf67d1 100644
--- a/apps/docs/resources/globalSearch/globalSearchModel.ts
+++ b/apps/docs/resources/globalSearch/globalSearchModel.ts
@@ -15,6 +15,8 @@ import { ReferenceSDKFunctionModel, SDKLanguageValues } from '../reference/refer
import { TroubleshootingModel } from '../troubleshooting/troubleshootingModel'
import { SearchResultInterface } from './globalSearchInterface'
+const KB_RESULTS = 5
+
type SearchFunction = 'search_content' | 'search_content_nimbus'
type SearchHybridFunction = 'search_content_hybrid' | 'search_content_hybrid_nimbus'
@@ -57,6 +59,9 @@ export abstract class SearchResultModel {
): Promise<Result<SearchResultModel[], ApiErrorGeneric>> {
const query = args.query.trim()
const includeFullContent = requestedFields.includes('content')
+ const limit = args.limit ?? 30
+ const kbLimit = Math.min(Math.max(limit - KB_RESULTS, 0), KB_RESULTS)
+ const kbSearch = searchKb(query, kbLimit)
const embeddingResult = await openAI().createContentEmbedding(query)
const useAltSearchIndex = !isFeatureEnabled('search:fullIndex')
@@ -64,13 +69,13 @@ export abstract class SearchResultModel {
? 'search_content_hybrid_nimbus'
: 'search_content_hybrid'
- return embeddingResult.flatMapAsync(async ({ embedding }) => {
+ const docsResult = await embeddingResult.flatMapAsync(async ({ embedding }) => {
const matchResult = new Result(
await supabase().rpc(searchFunction, {
query_text: query,
query_embedding: embedding,
include_full_content: includeFullContent,
- max_result: args.limit ?? 30,
+ max_result: limit - kbLimit,
})
)
.map((matches) =>
@@ -82,6 +87,30 @@ export abstract class SearchResultModel {
return matchResult
})
+
+ const kbResults = await kbSearch
+ return docsResult.map((items) => [...items, ...kbResults])
+ }
+}
+
+/** Fails open to no results so docs search keeps working if KB is slow or down. */
+async function searchKb(query: string, limit: number): Promise<SearchResultInterface[]> {
+ if (limit <= 0) return []
+
+ try {
+ const params = new URLSearchParams({ q: query, limit: String(limit), content: 'true' })
+ const response = await fetch(`https://supabase.com/kb/api/search?${params}`, {
+ signal: AbortSignal.timeout(3000),
+ })
+ if (!response.ok) throw new Error(`KB search responded with ${response.status}`)
+
+ return (await response.json()).map(({ title, href, content }: Record<string, string>) => {
+ const Model = href.includes('/kb/troubleshooting/') ? TroubleshootingModel : GuideModel
+ return new Model({ title, href, content: content ?? undefined })
+ })
+ } catch (error) {
+ console.warn('Error searching KB:', error)
+ return []
}
}
diff --git a/apps/docs/scripts/search_v2/ingest.ts b/apps/docs/scripts/search_v2/ingest.ts
index 89767484b7e..6c94a83b961 100644
--- a/apps/docs/scripts/search_v2/ingest.ts
+++ b/apps/docs/scripts/search_v2/ingest.ts
@@ -1,112 +1,20 @@
import '../utils/dotenv.js'
-import { readFile } from 'node:fs/promises'
import path from 'node:path'
-import { fileURLToPath } from 'node:url'
-import fg from 'fast-glob'
+import { ingestMarkdown } from 'docs-search/ingest'
-import { parsePage } from './markdown.js'
-import { filePathToSlug } from './routes.js'
import { createSupabaseClient, TABLE_NAME } from './supabase.js'
-// ---------------------------------------------------------------------------
-// Configuration
-// ---------------------------------------------------------------------------
-
-/** Folder that holds the markdown files. Searched recursively. */
-export const CONTENT_ROOT = path.resolve(process.cwd(), 'public/markdown')
-
-/** Glob patterns (relative to CONTENT_ROOT) for the files to ingest. */
-const FILE_PATTERNS = ['**/*.md', '**/*.mdx']
-
-/** Glob patterns (relative to CONTENT_ROOT) for files/folders to skip. */
-const IGNORED_PATTERNS = ['reference/**', 'guides/troubleshooting/**', '_partials/**']
-
-/** How many rows to send to Supabase per insert request. */
-const INSERT_BATCH_SIZE = 200
-
-// ---------------------------------------------------------------------------
-
-interface SectionRow {
- slug: string
- file_path: string
- page_title: string
- heading: string
- heading_level: number
- heading_path: string[]
- content: string
- excerpt: string
-}
-
-/** Build the DB rows for a single markdown file. Pure: no I/O. */
-export function buildRows(markdown: string, filePath: string, contentRoot: string): SectionRow[] {
- const slug = filePathToSlug(filePath, contentRoot)
- const relativePath = path.relative(contentRoot, filePath).split(path.sep).join('/')
- const page = parsePage(markdown)
- const pageTitle = page.title || path.basename(slug) || 'Untitled'
-
- return page.sections.map((section) => ({
- slug,
- file_path: relativePath,
- page_title: pageTitle,
- heading: section.heading,
- heading_level: section.level,
- heading_path: section.headingPath,
- content: section.content,
- excerpt: page.excerpt,
- }))
-}
-
-function chunk<T>(items: T[], size: number): T[][] {
- const out: T[][] = []
- for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))
- return out
-}
-
async function main() {
- const supabase = createSupabaseClient()
-
- const files = await fg(FILE_PATTERNS, {
- cwd: CONTENT_ROOT,
- absolute: true,
- onlyFiles: true,
- ignore: IGNORED_PATTERNS,
+ await ingestMarkdown({
+ supabase: createSupabaseClient(),
+ table: TABLE_NAME,
+ contentRoot: path.resolve(process.cwd(), 'public/markdown'),
+ ignore: ['reference/**', 'guides/troubleshooting/**', '_partials/**'],
})
- if (files.length === 0) {
- console.warn(`No markdown files found under ${CONTENT_ROOT}`)
- return
- }
- console.log(`Found ${files.length} markdown file(s) under ${CONTENT_ROOT}`)
-
- let totalRows = 0
-
- for (const file of files.sort()) {
- const markdown = await readFile(file, 'utf8')
- const rows = buildRows(markdown, file, CONTENT_ROOT)
- const slug = rows[0]?.slug ?? filePathToSlug(file, CONTENT_ROOT)
-
- // Replace everything previously ingested for this page.
- const { error: deleteError } = await supabase.from(TABLE_NAME).delete().eq('slug', slug)
- if (deleteError) throw new Error(`Failed to clear "${slug}": ${deleteError.message}`)
-
- for (const batch of chunk(rows, INSERT_BATCH_SIZE)) {
- const { error: insertError } = await supabase.from(TABLE_NAME).insert(batch)
- if (insertError) throw new Error(`Failed to insert "${slug}": ${insertError.message}`)
- }
-
- totalRows += rows.length
- console.log(` ✓ ${slug || '(root)'} (${rows.length} section${rows.length === 1 ? '' : 's'})`)
- }
-
- console.log(`Done. Ingested ${totalRows} section(s) from ${files.length} file(s).`)
}
-// Only run when executed directly (so buildRows can be imported by tests).
-const isDirectRun =
- !!process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)
-if (isDirectRun) {
- main().catch((err) => {
- console.error(err instanceof Error ? err.message : err)
- process.exit(1)
- })
-}
+main().catch((err) => {
+ console.error(err instanceof Error ? err.message : err)
+ process.exit(1)
+})
diff --git a/apps/docs/scripts/search_v2/sql/setup.sql b/apps/docs/scripts/search_v2/sql/setup.sql
index 203f1212183..09ef0b1acf5 100644
--- a/apps/docs/scripts/search_v2/sql/setup.sql
+++ b/apps/docs/scripts/search_v2/sql/setup.sql
@@ -1,5 +1,5 @@
--- Run this in the Supabase SQL editor before `pnpm search-v2:ingest`.
+-- Run this in the Supabase SQL editor before `pnpm ingest:search_v2`.
-- Safe to re-run: it uses IF NOT EXISTS / CREATE OR REPLACE.
-- One row per markdown *section* (a heading + the text below it, up to the next heading).
diff --git a/apps/kb/.env.example b/apps/kb/.env.example
index eed0248c52f..49d09643de9 100644
--- a/apps/kb/.env.example
+++ b/apps/kb/.env.example
@@ -15,6 +15,7 @@ GH_TOKEN=
# Supabase project backing troubleshooting_entries (same project apps/www and
# apps/studio use — supabase/config.toml's `[remotes.prod]`). Everything in kb
# uses the service role key: the Astro build reads with it, the sync scripts
-# also write. Never expose it to client code (kb is static, build-time only).
+# also write, and api/search.ts queries kb_search at runtime. Never
+# expose it to client code.
SUPABASE_URL=
SUPABASE_SERVICE_ROLE_KEY=
diff --git a/apps/kb/api/search.ts b/apps/kb/api/search.ts
new file mode 100644
index 00000000000..0c6a57de7f8
--- /dev/null
+++ b/apps/kb/api/search.ts
@@ -0,0 +1,75 @@
+import { createClient, type SupabaseClient } from '@supabase/supabase-js'
+
+import { corsHeaders } from '../src/lib/search/cors'
+import type { Database, KbSearchResult } from '../src/lib/search/types'
+
+let client: SupabaseClient<Database> | undefined
+function searchClient() {
+ client ??= createClient<Database>(
+ process.env.SUPABASE_URL!,
+ process.env.SUPABASE_SERVICE_ROLE_KEY!,
+ { auth: { autoRefreshToken: false, persistSession: false, detectSessionInUrl: false } }
+ )
+ return client
+}
+
+/** Fetches the markdown export of a page (`<href>.md`); null if it can't be loaded. */
+async function fetchContent(href: string): Promise<string | null> {
+ try {
+ const response = await fetch(`${href}.md`)
+ if (!response.ok) {
+ console.warn(`Failed to fetch kb content from ${href}.md: ${response.status}`)
+ return null
+ }
+ return await response.text()
+ } catch (error) {
+ console.warn(`Failed to fetch kb content from ${href}.md:`, error)
+ return null
+ }
+}
+
+export function OPTIONS() {
+ return new Response(null, { headers: corsHeaders })
+}
+
+export async function GET(request: Request) {
+ const { searchParams } = new URL(request.url)
+ const query = searchParams.get('q')?.trim()
+ const limit = Number(searchParams.get('limit')) || 10
+ const includeContent = searchParams.get('content') === 'true'
+ if (!query)
+ return Response.json({ error: 'Missing q parameter' }, { status: 400, headers: corsHeaders })
+
+ const { data, error } = await searchClient().rpc('search_kb', {
+ query_text: query,
+ match_limit: limit,
+ })
+ if (error) {
+ console.error('Error running kb search:', error)
+ return Response.json(
+ { error: 'There was an error processing your request' },
+ { status: 500, headers: corsHeaders }
+ )
+ }
+
+ let results: KbSearchResult[] = data.map((row) => ({
+ title: row.page_title,
+ href: `https://supabase.com/${row.slug.toLowerCase()}`,
+ heading: row.heading,
+ excerpt: row.excerpt,
+ score: row.score,
+ }))
+
+ if (includeContent) {
+ results = await Promise.all(
+ results.map(async (result) => ({ ...result, content: await fetchContent(result.href) }))
+ )
+ }
+
+ return Response.json(results, {
+ headers: {
+ ...corsHeaders,
+ 'Cache-Control': 'public, s-maxage=3600, stale-while-revalidate=300',
+ },
+ })
+}
diff --git a/apps/kb/package.json b/apps/kb/package.json
index ccc92ca4bb9..9e0135c48bd 100644
--- a/apps/kb/package.json
+++ b/apps/kb/package.json
@@ -11,6 +11,8 @@
"build": "astro build",
"fetch:federated-content": "tsx ./scripts/federated-content/fetch-federated-content.ts",
"generate:markdown": "tsx ./scripts/generate-markdown.mjs",
+ "presearch:ingest": "pnpm run fetch:federated-content && pnpm run generate:markdown",
+ "search:ingest": "tsx ./scripts/search/ingest.ts",
"sync:legacy-slugs": "tsx ./scripts/federated-content/backfill-legacy-slugs.ts",
"sync:troubleshooting-entries": "tsx ./scripts/federated-content/sync-troubleshooting-entries.ts",
"sync:troubleshooting-updates": "tsx ./scripts/federated-content/sync-troubleshooting-updates.ts",
@@ -29,6 +31,7 @@
"@supabase/supabase-js": "catalog:",
"astro": "^7.2.6",
"common": "workspace:*",
+ "docs-search": "workspace:*",
"gray-matter": "^4.0.3",
"lucide-react": "catalog:",
"react": "catalog:",
diff --git a/apps/kb/scripts/search/ingest.ts b/apps/kb/scripts/search/ingest.ts
new file mode 100644
index 00000000000..79bb958d2e8
--- /dev/null
+++ b/apps/kb/scripts/search/ingest.ts
@@ -0,0 +1,26 @@
+import '../utils/dotenv.js'
+
+import path from 'node:path'
+import { createClient } from '@supabase/supabase-js'
+import { ingestMarkdown } from 'docs-search/ingest'
+
+import { TABLE_NAME } from './supabase.js'
+
+const { SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY } = process.env
+if (!SUPABASE_URL || !SUPABASE_SERVICE_ROLE_KEY) {
+ console.error('Missing SUPABASE_URL or SUPABASE_SERVICE_ROLE_KEY. Set them in .env.local.')
+ process.exit(1)
+}
+
+ingestMarkdown({
+ supabase: createClient(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY, {
+ auth: { autoRefreshToken: false, persistSession: false },
+ }),
+ table: TABLE_NAME,
+ contentRoot: path.resolve(process.cwd(), 'public/markdown'),
+ include: ['**/*.md'],
+ slugPrefix: 'kb/',
+}).catch((err) => {
+ console.error(err instanceof Error ? err.message : err)
+ process.exit(1)
+})
diff --git a/apps/kb/scripts/search/supabase.ts b/apps/kb/scripts/search/supabase.ts
new file mode 100644
index 00000000000..543874c0adb
--- /dev/null
+++ b/apps/kb/scripts/search/supabase.ts
@@ -0,0 +1 @@
+export const TABLE_NAME = 'kb_search'
diff --git a/apps/kb/src/lib/search/cors.ts b/apps/kb/src/lib/search/cors.ts
new file mode 100644
index 00000000000..c7243dfacd1
--- /dev/null
+++ b/apps/kb/src/lib/search/cors.ts
@@ -0,0 +1,5 @@
+export const corsHeaders = {
+ 'Access-Control-Allow-Origin': '*',
+ 'Access-Control-Allow-Methods': 'GET, OPTIONS',
+ 'Access-Control-Allow-Headers': 'content-type',
+}
diff --git a/apps/kb/src/lib/search/types.ts b/apps/kb/src/lib/search/types.ts
new file mode 100644
index 00000000000..07c11e69d34
--- /dev/null
+++ b/apps/kb/src/lib/search/types.ts
@@ -0,0 +1,33 @@
+export interface KbSearchRow {
+ slug: string
+ page_title: string
+ heading: string
+ heading_level: number
+ heading_path: string[]
+ excerpt: string
+ score: number
+}
+
+export interface KbSearchResult {
+ title: string
+ href: string
+ heading: string
+ excerpt: string
+ score: number
+ content?: string | null
+}
+
+export type Database = {
+ public: {
+ Tables: Record<string, never>
+ Views: Record<string, never>
+ Functions: {
+ search_kb: {
+ Args: { query_text: string; match_limit?: number }
+ Returns: KbSearchRow[]
+ }
+ }
+ Enums: Record<string, never>
+ CompositeTypes: Record<string, never>
+ }
+}
diff --git a/package.json b/package.json
index 1a77eeabed3..7d806865f65 100644
--- a/package.json
+++ b/package.json
@@ -27,7 +27,7 @@
"test:case-hazards": "node scripts/check-case-hazards.mjs",
"test:prettier": "SORT_IMPORTS=false prettier --config prettier.config.mjs --cache --check '{apps,packages,blocks,examples,i18n}/**/*.{js,jsx,ts,tsx,css,md,mdx,json}'",
"format": "SORT_IMPORTS=false prettier --config prettier.config.mjs --cache --write '{apps,packages,blocks,examples,i18n}/**/*.{js,jsx,ts,tsx,css,md,mdx,json}'",
- "test:docs": "turbo run test --filter=docs",
+ "test:docs": "turbo run test --filter=docs --filter=docs-search",
"test:kb": "turbo run test --filter=kb",
"test:ui": "turbo run test --filter=ui",
"test:ui-patterns": "turbo run test --filter=ui-patterns",
diff --git a/packages/docs-search/package.json b/packages/docs-search/package.json
new file mode 100644
index 00000000000..fcd7fe66368
--- /dev/null
+++ b/packages/docs-search/package.json
@@ -0,0 +1,31 @@
+{
+ "name": "docs-search",
+ "version": "0.0.0",
+ "license": "MIT",
+ "scripts": {
+ "preinstall": "npx only-allow pnpm",
+ "clean": "rimraf .turbo tsconfig.tsbuildinfo",
+ "typecheck": "tsc --noEmit",
+ "test": "vitest --run"
+ },
+ "dependencies": {
+ "fast-glob": "^3.3.3",
+ "mdast-util-to-string": "^3.1.1",
+ "remark-frontmatter": "^5.0.0",
+ "remark-gfm": "^4.0.0",
+ "remark-parse": "^11.0.0",
+ "unified": "^11.0.5"
+ },
+ "devDependencies": {
+ "@types/mdast": "^3.0.0",
+ "@types/node": "catalog:",
+ "@typescript/native": "catalog:",
+ "tsconfig": "workspace:*",
+ "typescript": "catalog:",
+ "vitest": "catalog:",
+ "@supabase/supabase-js": "catalog:"
+ },
+ "exports": {
+ "./ingest": "./src/ingest.ts"
+ }
+}
diff --git a/packages/docs-search/src/ingest.ts b/packages/docs-search/src/ingest.ts
new file mode 100644
index 00000000000..00a398270e9
--- /dev/null
+++ b/packages/docs-search/src/ingest.ts
@@ -0,0 +1,52 @@
+import { readFile } from 'node:fs/promises'
+import fg from 'fast-glob'
+
+import { filePathToSlug } from './routes.js'
+import { buildRows } from './rows.js'
+import type { IngestOptions } from './types.js'
+
+const INSERT_BATCH_SIZE = 200
+
+function chunk<T>(items: T[], size: number): T[][] {
+ const out: T[][] = []
+ for (let i = 0; i < items.length; i += size) out.push(items.slice(i, i + size))
+ return out
+}
+
+export async function ingestMarkdown({
+ supabase,
+ contentRoot,
+ include = ['**/*.md', '**/*.mdx'],
+ ignore = [],
+ table,
+ slugPrefix = '',
+}: IngestOptions): Promise<number> {
+ const files = await fg(include, { cwd: contentRoot, absolute: true, onlyFiles: true, ignore })
+ if (files.length === 0) {
+ console.warn(`No markdown files found under ${contentRoot}`)
+ return 0
+ }
+ console.log(`Found ${files.length} markdown file(s) under ${contentRoot}`)
+
+ let totalRows = 0
+
+ for (const file of files.sort()) {
+ const markdown = await readFile(file, 'utf8')
+ const rows = buildRows(markdown, file, contentRoot, slugPrefix)
+ const slug = rows[0]?.slug ?? slugPrefix + filePathToSlug(file, contentRoot)
+
+ const { error: deleteError } = await supabase.from(table).delete().eq('slug', slug)
+ if (deleteError) throw new Error(`Failed to clear "${slug}": ${deleteError.message}`)
+
+ for (const batch of chunk(rows, INSERT_BATCH_SIZE)) {
+ const { error: insertError } = await supabase.from(table).insert(batch)
+ if (insertError) throw new Error(`Failed to insert "${slug}": ${insertError.message}`)
+ }
+
+ totalRows += rows.length
+ console.log(` ✓ ${slug || '(root)'} (${rows.length} section${rows.length === 1 ? '' : 's'})`)
+ }
+
+ console.log(`Done. Ingested ${totalRows} section(s) from ${files.length} file(s).`)
+ return totalRows
+}
diff --git a/apps/docs/scripts/search_v2/markdown.test.ts b/packages/docs-search/src/markdown.test.ts
similarity index 100%
rename from apps/docs/scripts/search_v2/markdown.test.ts
rename to packages/docs-search/src/markdown.test.ts
diff --git a/apps/docs/scripts/search_v2/markdown.ts b/packages/docs-search/src/markdown.ts
similarity index 71%
rename from apps/docs/scripts/search_v2/markdown.ts
rename to packages/docs-search/src/markdown.ts
index 7fb4c26d334..a247efa2186 100644
--- a/apps/docs/scripts/search_v2/markdown.ts
+++ b/packages/docs-search/src/markdown.ts
@@ -4,39 +4,20 @@ import remarkGfm from 'remark-gfm'
import remarkParse from 'remark-parse'
import { unified } from 'unified'
+import type { ParsedPage, Section } from './types.js'
+
const processor = unified().use(remarkParse).use(remarkFrontmatter, ['yaml']).use(remarkGfm)
/**
- * Types are derived from the processor's own output rather than imported from
- * `mdast` directly: this app pins `@types/mdast` v3 for its wider MDX pipeline,
- * while `remark-parse` v11 resolves its own (structurally incompatible) v4
- * types for its nested `@types/mdast` dependency.
+ * Types are derived from the processor's output rather than imported from `mdast`:
+ * the docs app pins `@types/mdast` v3, while `remark-parse` v11 resolves its own
+ * (structurally incompatible) v4 types.
*/
export type Root = ReturnType<typeof processor.parse>
-/** Any child of the root node (paragraph, heading, list, table, ...). */
type Content = Root['children'][number]
type Heading = Extract<Content, { type: 'heading' }>
-/** Union of every node type that can appear in the tree (root or content). */
type Nodes = Root | Content
-export interface Section {
- /** Text of this section's heading ('' for text that appears before any heading). */
- heading: string
- /** 1..6, or 0 for text before any heading. */
- level: number
- /** Headings from the H1 down to this section, e.g. ['Users', 'Permanent and anonymous users']. */
- headingPath: string[]
- /** Plain text of everything under the heading until the next heading. */
- content: string
-}
-
-export interface ParsedPage {
- title: string
- excerpt: string
- sections: Section[]
-}
-
-/** Parse markdown into an mdast tree (GFM tables/lists + YAML frontmatter supported). */
export function parseMarkdownAst(markdown: string): Root {
return processor.parse(markdown)
}
@@ -59,7 +40,6 @@ export function nodeToText(node: Nodes): string {
}
}
-/** The first H1 of the document, or '' if there is none. */
export function extractTitle(tree: Root): string {
const h1 = tree.children.find(
(node): node is Heading => node.type === 'heading' && node.depth === 1
@@ -73,7 +53,6 @@ export function extractExcerpt(tree: Root): string {
return paragraph ? nodeToText(paragraph) : ''
}
-/** Nodes that carry no searchable text. */
function isIgnored(node: Content): boolean {
return (
node.type === 'yaml' ||
@@ -118,7 +97,6 @@ export function extractSections(tree: Root): Section[] {
continue
}
- // Body text before the first heading gets a headless section.
if (!current) current = { heading: '', level: 0, headingPath: [], content: '' }
const text = nodeToText(node)
if (text) body.push(text)
@@ -128,7 +106,6 @@ export function extractSections(tree: Root): Section[] {
return sections
}
-/** Everything the ingester needs from one markdown file. */
export function parsePage(markdown: string): ParsedPage {
const tree = parseMarkdownAst(markdown)
return {
diff --git a/apps/docs/scripts/search_v2/routes.test.ts b/packages/docs-search/src/routes.test.ts
similarity index 100%
rename from apps/docs/scripts/search_v2/routes.test.ts
rename to packages/docs-search/src/routes.test.ts
diff --git a/apps/docs/scripts/search_v2/routes.ts b/packages/docs-search/src/routes.ts
similarity index 100%
rename from apps/docs/scripts/search_v2/routes.ts
rename to packages/docs-search/src/routes.ts
diff --git a/packages/docs-search/src/rows.ts b/packages/docs-search/src/rows.ts
new file mode 100644
index 00000000000..419c230df1f
--- /dev/null
+++ b/packages/docs-search/src/rows.ts
@@ -0,0 +1,28 @@
+import path from 'node:path'
+
+import { parsePage } from './markdown.js'
+import { filePathToSlug } from './routes.js'
+import type { SectionRow } from './types.js'
+
+export function buildRows(
+ markdown: string,
+ filePath: string,
+ contentRoot: string,
+ slugPrefix = ''
+): SectionRow[] {
+ const slug = slugPrefix + filePathToSlug(filePath, contentRoot)
+ const relativePath = path.relative(contentRoot, filePath).split(path.sep).join('/')
+ const page = parsePage(markdown)
+ const pageTitle = page.title || path.basename(slug) || 'Untitled'
+
+ return page.sections.map((section) => ({
+ slug,
+ file_path: relativePath,
+ page_title: pageTitle,
+ heading: section.heading,
+ heading_level: section.level,
+ heading_path: section.headingPath,
+ content: section.content,
+ excerpt: page.excerpt,
+ }))
+}
diff --git a/packages/docs-search/src/types.ts b/packages/docs-search/src/types.ts
new file mode 100644
index 00000000000..66ade5c875b
--- /dev/null
+++ b/packages/docs-search/src/types.ts
@@ -0,0 +1,34 @@
+import type { SupabaseClient } from '@supabase/supabase-js'
+
+export interface Section {
+ heading: string
+ level: number
+ headingPath: string[]
+ content: string
+}
+
+export interface ParsedPage {
+ title: string
+ excerpt: string
+ sections: Section[]
+}
+
+export interface SectionRow {
+ slug: string
+ file_path: string
+ page_title: string
+ heading: string
+ heading_level: number
+ heading_path: string[]
+ content: string
+ excerpt: string
+}
+
+export interface IngestOptions {
+ supabase: SupabaseClient
+ contentRoot: string
+ include?: string[]
+ ignore?: string[]
+ table: string
+ slugPrefix?: string
+}
diff --git a/packages/docs-search/tsconfig.json b/packages/docs-search/tsconfig.json
new file mode 100644
index 00000000000..2425077f19f
--- /dev/null
+++ b/packages/docs-search/tsconfig.json
@@ -0,0 +1,9 @@
+{
+ "extends": "tsconfig/react-library.json",
+ "include": ["."],
+ "exclude": ["dist", "build", "node_modules"],
+ "compilerOptions": {
+ "declaration": false,
+ "declarationMap": false
+ }
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 06b006b48d8..07c70f0b5b6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -422,6 +422,9 @@ importers:
dev-tools:
specifier: workspace:*
version: link:../../packages/dev-tools
+ docs-search:
+ specifier: workspace:*
+ version: link:../../packages/docs-search
eslint-config-supabase:
specifier: workspace:*
version: link:../../packages/eslint-config-supabase
@@ -774,6 +777,9 @@ importers:
common:
specifier: workspace:*
version: link:../../packages/common
+ docs-search:
+ specifier: workspace:*
+ version: link:../../packages/docs-search
gray-matter:
specifier: ^4.0.3
version: 4.0.3
@@ -2560,6 +2566,49 @@ importers:
specifier: 'catalog:'
+ packages/docs-search:
+ dependencies:
+ fast-glob:
+ specifier: ^3.3.3
+ version: 3.3.3
+ mdast-util-to-string:
+ specifier: ^3.1.1
+ version: 3.2.0
+ remark-frontmatter:
+ specifier: ^5.0.0
+ version: 5.0.0([email protected])
+ remark-gfm:
+ specifier: ^4.0.0
+ version: 4.0.1([email protected])
+ remark-parse:
+ specifier: ^11.0.0
+ version: 11.0.0([email protected])
+ unified:
+ specifier: ^11.0.5
+ version: 11.0.5
+ devDependencies:
+ '@supabase/supabase-js':
+ specifier: 'catalog:'
+ version: 2.117.3(@opentelemetry/[email protected])
+ '@types/mdast':
+ specifier: ^3.0.0
+ version: 3.0.15
+ '@types/node':
+ specifier: 'catalog:'
+ version: 22.13.14
+ '@typescript/native':
+ specifier: 'catalog:'
+ tsconfig:
+ specifier: workspace:*
+ version: link:../tsconfig
+ typescript:
+ specifier: 'catalog:'
+ version: 6.0.2
+ vitest:
+ specifier: 'catalog:'
+
packages/eslint-config-supabase:
devDependencies:
'@shadcn/lint':
diff --git a/supabase/migrations/20261016120000_create_kb_search.sql b/supabase/migrations/20261016120000_create_kb_search.sql
new file mode 100644
index 00000000000..670d8ac0351
--- /dev/null
+++ b/supabase/migrations/20261016120000_create_kb_search.sql
@@ -0,0 +1,88 @@
+create table public.kb_search (
+ id bigint generated always as identity primary key,
+ slug text not null,
+ file_path text not null,
+ page_title text not null default '',
+ heading text not null default '',
+ heading_level smallint not null default 0,
+ heading_path text[] not null default '{}',
+ content text not null default '',
+ excerpt text not null default '',
+ updated_at timestamptz not null default now(),
+ fts tsvector generated always as (
+ setweight(to_tsvector('english', coalesce(page_title, '')), 'A')
+ || setweight(
+ to_tsvector('english', coalesce(heading, '')),
+ (case heading_level when 1 then 'A' when 2 then 'B' when 3 then 'C' else 'D' end)::"char"
+ )
+ || setweight(to_tsvector('english', coalesce(content, '')), 'D')
+ ) stored
+);
+
+create index kb_search_fts_idx on public.kb_search using gin (fts);
+create index kb_search_slug_idx on public.kb_search (slug);
+
+alter table public.kb_search enable row level security;
+
+grant select, insert, delete on public.kb_search to service_role;
+
+create function public.search_kb(query_text text, match_limit int default 10)
+returns table (
+ slug text,
+ page_title text,
+ heading text,
+ heading_level smallint,
+ heading_path text[],
+ excerpt text,
+ score int
+)
+language sql
+stable
+set search_path = ''
+as $$
+ with terms as (
+ select distinct
+ to_tsquery('simple', quote_literal(lexeme)) as anywhere,
+ to_tsquery('simple', quote_literal(lexeme) || ':AB') as in_title
+ from unnest(tsvector_to_array(to_tsvector('english', query_text))) as lexeme
+ ),
+ weighted_terms as (
+ select
+ t.anywhere,
+ t.in_title,
+ greatest(0, ln(
+ (select count(*) from public.kb_search)::float
+ / (1 + (select count(*) from public.kb_search s where s.fts @@ t.anywhere))
+ )) as idf
+ from terms t
+ ),
+ ranked as (
+ select
+ s.slug,
+ s.page_title,
+ s.heading,
+ s.heading_level,
+ s.heading_path,
+ s.excerpt,
+ round(
+ 1000 * sum(w.idf * case when s.fts @@ w.in_title then 2 else 1 end)
+ / nullif((select 2 * sum(idf) from weighted_terms), 0)
+ )::int as score
+ from public.kb_search s
+ join weighted_terms w on s.fts @@ w.anywhere
+ group by s.id
+ ),
+ best_per_page as (
+ select distinct on (r.slug) r.*
+ from ranked r
+ where r.score >= 300
+ order by r.slug, r.score desc, r.heading_level asc
+ )
+ select b.slug, b.page_title, b.heading, b.heading_level, b.heading_path, b.excerpt, b.score
+ from best_per_page b
+ order by b.score desc, b.heading_level asc, b.slug asc
+ limit greatest(1, least(coalesce(match_limit, 10), 100));
+$$;
+
+revoke execute on function public.search_kb(text, int) from public, anon, authenticated;
+grant execute on function public.search_kb(text, int) to service_role;