Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(postgres): support pgsearch keyword scoring (#4502)
bert pan9110cb2da170
diff --git a/internal/application/repository/retriever/postgres/keyword_score_integration_test.go b/internal/application/repository/retriever/postgres/keyword_score_integration_test.gonew file mode 100644index 000000000..666bf7132--- /dev/null+++ b/internal/application/repository/retriever/postgres/keyword_score_integration_test.go@@ -0,0 +1,104 @@+package postgres++import (+ "context"+ "os"+ "testing"++ "github.com/stretchr/testify/require"+ gormpostgres "gorm.io/driver/postgres"+ "gorm.io/gorm"+ "gorm.io/gorm/logger"+)++// TestKeywordScoreCatalogCompatibility checks the actual PostgreSQL catalog+// query, including preference and signatures, rather than mocking its result.+// Set WEKNORA_RETRIEVER_TEST_POSTGRES_DSN to a disposable PostgreSQL database+// without pg_search installed. Fixtures are rolled back after each subtest;+// existing scoring schemas are never replaced or dropped.+func TestKeywordScoreCatalogCompatibility(t *testing.T) {+ dsn := os.Getenv("WEKNORA_RETRIEVER_TEST_POSTGRES_DSN")+ if dsn == "" {+ t.Skip("set WEKNORA_RETRIEVER_TEST_POSTGRES_DSN to run PostgreSQL catalog tests")+ }+ db, err := gorm.Open(gormpostgres.Open(dsn), &gorm.Config{+ Logger: logger.Default.LogMode(logger.Silent),+ })+ require.NoError(t, err)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { _ = sqlDB.Close() })++ var existing int+ require.NoError(t, db.Raw(`SELECT count(*) FROM pg_catalog.pg_namespace+ WHERE nspname IN ('paradedb', 'pgsearch', 'unrelated_score_test')`).Scan(&existing).Error)+ require.Zero(t, existing, "use a disposable database without existing scoring schemas")++ paradedb := "CREATE FUNCTION paradedb.score(anyelement) RETURNS real LANGUAGE sql AS 'SELECT 1::real'"+ pgsearch := "CREATE FUNCTION pgsearch.score(anyelement) RETURNS real LANGUAGE sql AS 'SELECT 1::real'"+ for _, tc := range []struct {+ name string+ functions []string+ want string+ }{+ {name: "ParadeDB", functions: []string{paradedb}, want: "paradedb.score"},+ {name: "PolarDB_layout", functions: []string{pgsearch}, want: "pgsearch.score"},+ {name: "prefer_ParadeDB", functions: []string{pgsearch, paradedb}, want: "paradedb.score"},+ {name: "missing_functions"},+ {+ name: "incompatible_argument_falls_back",+ functions: []string{+ "CREATE FUNCTION paradedb.score(integer) RETURNS real LANGUAGE sql AS 'SELECT 1::real'",+ pgsearch,+ },+ want: "pgsearch.score",+ },+ {+ name: "incompatible_result_falls_back",+ functions: []string{+ "CREATE FUNCTION paradedb.score(anyelement) RETURNS double precision LANGUAGE sql AS 'SELECT 1::real'",+ pgsearch,+ },+ want: "pgsearch.score",+ },+ {+ name: "reject_wrong_arity",+ functions: []string{+ "CREATE FUNCTION pgsearch.score(anyelement, integer) RETURNS real LANGUAGE sql AS 'SELECT 1::real'",+ },+ },+ {+ name: "reject_unlisted_schema",+ functions: []string{+ "CREATE FUNCTION unrelated_score_test.score(anyelement) RETURNS real LANGUAGE sql AS 'SELECT 1::real'",+ },+ },+ {+ name: "reject_wrong_result",+ functions: []string{+ "CREATE FUNCTION pgsearch.score(anyelement) RETURNS double precision LANGUAGE sql AS 'SELECT 1::real'",+ },+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ tx := db.Begin()+ require.NoError(t, tx.Error)+ t.Cleanup(func() { require.NoError(t, tx.Rollback().Error) })+ for _, schema := range []string{"paradedb", "pgsearch", "unrelated_score_test"} {+ require.NoError(t, tx.Exec("CREATE SCHEMA "+schema).Error)+ }+ for _, ddl := range tc.functions {+ require.NoError(t, tx.Exec(ddl).Error)+ }+ repo := &pgRepository{db: tx}+ function, err := repo.resolveKeywordScoreFunction(context.Background())+ if tc.want == "" {+ require.ErrorContains(t, err, "check pg_search extension compatibility")+ require.Empty(t, function)+ return+ }+ require.NoError(t, err)+ require.Equal(t, tc.want, function)+ })+ }+}diff --git a/internal/application/repository/retriever/postgres/keyword_score_test.go b/internal/application/repository/retriever/postgres/keyword_score_test.gonew file mode 100644index 000000000..22b0564d0--- /dev/null+++ b/internal/application/repository/retriever/postgres/keyword_score_test.go@@ -0,0 +1,193 @@+package postgres++import (+ "context"+ "errors"+ "regexp"+ "sync"+ "testing"++ "github.com/DATA-DOG/go-sqlmock"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+ gormpostgres "gorm.io/driver/postgres"+ "gorm.io/gorm"+ "gorm.io/gorm/logger"+)++func newKeywordTestRepository(t *testing.T) (*pgRepository, sqlmock.Sqlmock) {+ t.Helper()+ db, mock, err := sqlmock.New()+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, mock.ExpectationsWereMet()); _ = db.Close() })+ gdb, err := gorm.Open(gormpostgres.New(gormpostgres.Config{Conn: db}), &gorm.Config{+ DisableAutomaticPing: true,+ Logger: logger.Default.LogMode(logger.Silent),+ })+ require.NoError(t, err)+ return NewPostgresRetrieveEngineRepository(gdb).(*pgRepository), mock+}++// Exercise the actual query path for both backends, including scope filters,+// bound query text, score mapping, and reuse of the per-connection detection.+func TestKeywordsRetrieveScoreFunctionCompatibility(t *testing.T) {+ for _, scoreFunction := range []string{"paradedb.score", "pgsearch.score"} {+ t.Run(scoreFunction, func(t *testing.T) {+ repo, mock := newKeywordTestRepository(t)+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow(scoreFunction))+ params := types.RetrieveParams{+ RetrieverType: types.KeywordsRetrieverType,+ Query: "关键词测试 ' OR 1=1 --", TopK: 5,+ KnowledgeBaseIDs: []string{"kb-test"}, KnowledgeIDs: []string{"doc-test"},+ TagIDs: []string{"tag-test"},+ }+ query := `SELECT ` + regexp.QuoteMeta(scoreFunction) + `\(id\) as score,.*` ++ `FROM "embeddings" WHERE "knowledge_base_id" = \$1.*` ++ `"knowledge_id" = \$2.*"tag_id" = \$3.*` ++ `content \|\|\| \$4.*is_enabled IS NULL OR is_enabled = \$5.*` ++ `ORDER BY "score" DESC LIMIT \$6`+ for range 2 {+ mock.ExpectQuery(query).+ WithArgs("kb-test", "doc-test", "tag-test", params.Query, true, 5).+ WillReturnRows(sqlmock.NewRows([]string{+ "id", "chunk_id", "knowledge_id", "knowledge_base_id", "tag_id", "score",+ }).+ AddRow(1, "chunk-test", "doc-test", "kb-test", "tag-test", 31.15))+ results, err := repo.Retrieve(context.Background(), params)+ require.NoError(t, err)+ require.Len(t, results, 1)+ require.Len(t, results[0].Results, 1)+ require.Equal(t, "chunk-test", results[0].Results[0].ChunkID)+ require.Equal(t, 31.15, results[0].Results[0].Score)+ require.Equal(t, types.MatchTypeKeywords, results[0].Results[0].MatchType)+ }+ })+ }+}++func TestKeywordScoreDetectionFailuresDoNotPoisonCache(t *testing.T) {+ for _, kind := range []string{"missing", "unexpected", "database_error"} {+ t.Run(kind, func(t *testing.T) {+ repo, mock := newKeywordTestRepository(t)+ probe := mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL))+ cause := errors.New("catalog unavailable")+ switch kind {+ case "missing":+ probe.WillReturnRows(sqlmock.NewRows([]string{"score_function"}))+ case "unexpected":+ probe.WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("untrusted.score"))+ default:+ probe.WillReturnError(cause)+ }+ results, err := repo.KeywordsRetrieve(context.Background(), types.RetrieveParams{Query: "关键词测试", TopK: 5})+ require.Nil(t, results)+ if kind == "database_error" {+ require.ErrorIs(t, err, cause)+ } else {+ require.ErrorContains(t, err, "check pg_search extension compatibility")+ }+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("pgsearch.score"))+ function, err := repo.resolveKeywordScoreFunction(context.Background())+ require.NoError(t, err)+ require.Equal(t, "pgsearch.score", function)+ })+ }+}++func TestKeywordScoreDetectionConcurrentAndPerRepository(t *testing.T) {+ repo, mock := newKeywordTestRepository(t)+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("paradedb.score"))+ var wg sync.WaitGroup+ for range 16 {+ wg.Go(func() {+ function, err := repo.resolveKeywordScoreFunction(context.Background())+ if err != nil || function != "paradedb.score" {+ t.Errorf("resolve score function = %q, %v", function, err)+ }+ })+ }+ wg.Wait()+ other, otherMock := newKeywordTestRepository(t)+ otherMock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("pgsearch.score"))+ function, err := other.resolveKeywordScoreFunction(context.Background())+ require.NoError(t, err)+ require.Equal(t, "pgsearch.score", function)+}++func TestKeywordScoreDetectionCanceledContextCanRetry(t *testing.T) {+ repo, mock := newKeywordTestRepository(t)+ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ // A canceled caller must not cache a failed detection or block later calls.+ _, err := repo.resolveKeywordScoreFunction(ctx)+ require.ErrorIs(t, err, context.Canceled)+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("paradedb.score"))+ function, err := repo.resolveKeywordScoreFunction(context.Background())+ require.NoError(t, err)+ require.Equal(t, "paradedb.score", function)+}++func TestKeywordsRetrieveQueryFailurePreservesResolvedFunction(t *testing.T) {+ repo, mock := newKeywordTestRepository(t)+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("pgsearch.score"))+ query := `SELECT pgsearch\.score\(id\) as score,.*FROM "embeddings".*`+ cause := errors.New("keyword index unavailable")+ mock.ExpectQuery(query).WithArgs("关键词测试", true, 5).WillReturnError(cause)+ params := types.RetrieveParams{Query: "关键词测试", TopK: 5}+ results, err := repo.KeywordsRetrieve(context.Background(), params)+ require.Nil(t, results)+ require.ErrorIs(t, err, cause)++ // The catalog probe succeeded: retry only the retrieval, not detection.+ mock.ExpectQuery(query).WithArgs("关键词测试", true, 5).+ WillReturnRows(sqlmock.NewRows([]string{"id", "score"}))+ results, err = repo.KeywordsRetrieve(context.Background(), params)+ require.NoError(t, err)+ require.Len(t, results, 1)+ require.Empty(t, results[0].Results)+}++func TestVectorRetrieveDoesNotRequireKeywordScoreFunction(t *testing.T) {+ for _, keywordFailed := range []bool{false, true} {+ name := "without_keyword_probe"+ if keywordFailed {+ name = "after_unsupported_keyword_probe"+ }+ t.Run(name, func(t *testing.T) {+ repo, mock := newKeywordTestRepository(t)+ if keywordFailed {+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}))+ _, err := repo.KeywordsRetrieve(context.Background(), types.RetrieveParams{Query: "关键词测试", TopK: 5})+ require.Error(t, err)+ }++ // Expect only vector SQL: any score-function probe fails this test.+ mock.ExpectBegin()+ mock.ExpectExec("SET LOCAL hnsw.ef_search = 100").WillReturnResult(sqlmock.NewResult(0, 0))+ mock.ExpectExec("SET LOCAL hnsw.iterative_scan = strict_order").WillReturnResult(sqlmock.NewResult(0, 0))+ mock.ExpectQuery(`SELECT .*FROM embeddings.*`).+ WithArgs(sqlmock.AnyArg(), 2, true, 100, 0.5, 3).+ WillReturnRows(sqlmock.NewRows([]string{"id", "chunk_id", "score"}).AddRow(1, "vector-chunk", 0.9))+ mock.ExpectCommit()+ results, err := repo.Retrieve(context.Background(), types.RetrieveParams{+ RetrieverType: types.VectorRetrieverType,+ Embedding: []float32{0.1, 0.2},+ Threshold: 0.5,+ TopK: 3,+ })+ require.NoError(t, err)+ require.Len(t, results, 1)+ require.Len(t, results[0].Results, 1)+ require.Equal(t, "vector-chunk", results[0].Results[0].ChunkID)+ require.Equal(t, types.MatchTypeEmbedding, results[0].Results[0].MatchType)+ })+ }+}diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.goindex 4d1829ec7..cf28a9a27 100644--- a/internal/application/repository/retriever/postgres/repository.go+++ b/internal/application/repository/retriever/postgres/repository.go@@ -5,6 +5,7 @@ import ("errors""fmt""strings"+ "sync""github.com/Tencent/WeKnora/internal/common""github.com/Tencent/WeKnora/internal/logger"@@ -18,7 +19,9 @@ import (// pgRepository implements PostgreSQL-based retrieval operationstype pgRepository struct {- db *gorm.DB // Database connection+ db *gorm.DB // Database connection+ keywordScoreMu sync.Mutex+ keywordScoreFunction string}// NewPostgresRetrieveEngineRepository creates a new PostgreSQL retriever repository@@ -27,6 +30,38 @@ func NewPostgresRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineRreturn &pgRepository{db: db}}+// keywordScoreFunctionSQL only selects fixed, schema-qualified function names.+// Some PolarDB pg_search builds expose pgsearch.score instead of paradedb.score.+const keywordScoreFunctionSQL = `SELECT n.nspname || '.score' AS score_function+FROM pg_catalog.pg_proc p+JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace+WHERE n.nspname IN ('paradedb', 'pgsearch')+ AND p.proname = 'score' AND p.prokind = 'f' AND p.pronargs = 1+ AND p.proargtypes[0] = 'pg_catalog.anyelement'::regtype+ AND p.prorettype = 'pg_catalog.float4'::regtype+ORDER BY CASE n.nspname WHEN 'paradedb' THEN 0 ELSE 1 END+LIMIT 1`++// resolveKeywordScoreFunction probes lazily so vector-only stores remain usable.+// Cache successes per repository/database; transient failures can be retried.+func (g *pgRepository) resolveKeywordScoreFunction(ctx context.Context) (string, error) {+ g.keywordScoreMu.Lock()+ defer g.keywordScoreMu.Unlock()+ if g.keywordScoreFunction != "" {+ return g.keywordScoreFunction, nil+ }+ var row struct{ ScoreFunction string }+ if err := g.db.WithContext(ctx).Raw(keywordScoreFunctionSQL).Scan(&row).Error; err != nil {+ return "", fmt.Errorf("detect PostgreSQL keyword score function: %w", err)+ }+ if row.ScoreFunction != "paradedb.score" && row.ScoreFunction != "pgsearch.score" {+ return "", errors.New("PostgreSQL keyword retrieval requires paradedb.score(anyelement) or " ++ "pgsearch.score(anyelement); check pg_search extension compatibility")+ }+ g.keywordScoreFunction = row.ScoreFunction+ return g.keywordScoreFunction, nil+}+// EngineType returns the retriever engine type (PostgreSQL)func (r *pgRepository) EngineType() types.RetrieverEngineType {return types.PostgresRetrieverEngineType@@ -191,6 +226,10 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,params types.RetrieveParams,) ([]*types.RetrieveResult, error) {logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK)+ scoreFunction, err := g.resolveKeywordScoreFunction(ctx)+ if err != nil {+ return nil, err+ }conds := make([]clause.Expression, 0)// KnowledgeBaseIDs and KnowledgeIDs use AND logic@@ -236,9 +275,9 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,}})var embeddingDBList []pgVectorWithScore- err := g.db.WithContext(ctx).Clauses(conds...).+ err = g.db.WithContext(ctx).Clauses(conds...).Select([]string{- "paradedb.score(id) as score",+ scoreFunction + "(id) as score","id","content","source_id",diff --git a/website-docs/03-features/05-retrieval-engines.md b/website-docs/03-features/05-retrieval-engines.mdindex d544bc483..7286faea5 100644--- a/website-docs/03-features/05-retrieval-engines.md+++ b/website-docs/03-features/05-retrieval-engines.md@@ -23,7 +23,7 @@| 引擎 | RETRIEVE_DRIVER 值 | 向量检索 | 关键词/全文 | 关键词打分 | 中文分词 | 维度管理 | 阈值下推 | 部署复杂度 | 适用场景 ||------|-------------------|----------|------------|-----------|---------|----------|---------|-----------|----------|-| PostgreSQL | `postgres` | pgvector halfvec + HNSW 表达式索引 | ParadeDB BM25(`\|\|\|`) | BM25(paradedb.score) | ParadeDB tokenizer | 单表混维,表达式索引按维 cast | 距离阈值 SQL 内 | 低(默认镜像内置) | 默认选择;与业务同库,事务一致 |+| PostgreSQL | `postgres` | pgvector halfvec + HNSW 表达式索引 | pg_search BM25(`\|\|\|`) | BM25(优先 `paradedb.score`,兼容 `pgsearch.score`) | pg_search tokenizer | 单表混维,表达式索引按维 cast | 距离阈值 SQL 内 | 低(默认镜像内置) | 默认选择;与业务同库,事务一致 || SQLite | `sqlite` | sqlite-vec vec0(cosine) | FTS5 contentless | FTS5 | 应用侧 bigram | 每维一张 vec0 虚表 | 应用侧 | 极低(内嵌) | 桌面版 / 开发 / 微型部署 || Elasticsearch v8 | `elasticsearch_v8` | script_score cosineSimilarity | match(BM25) | BM25 | ES analyzer | dense_vector 单索引 | 应用侧 | 中 | 已有 ES 8 集群 || Elasticsearch v7 | `elasticsearch_v7` | 不支持(Support 仅 keywords) | match(BM25) | BM25 | ES analyzer | — | — | 中 | 存量 ES 7,仅作关键词引擎,需与其他向量引擎组合 |@@ -125,7 +125,8 @@ flowchart TD`internal/application/repository/retriever/postgres/repository.go`。数据与业务库同库(`embeddings` 表,GORM 管理)。- **向量检索**:pgvector `halfvec`(半精度,2 字节/维)。`embedding` 列不定维,HNSW 索引建在表达式 `(embedding::halfvec(dim)) halfvec_cosine_ops` 上——**ORDER BY 表达式必须与索引表达式完全一致**(两侧显式 cast),否则退化为顺序扫描(源码注释引 pgvector issue [#702](https://github.com/pgvector/pgvector/issues/702)/[#835](https://github.com/pgvector/pgvector/issues/835))。查询用子查询先取 `expandedTopK`(TopK*2,夹在 [100,200],避免大 LIMIT 拖垮 HNSW)个候选算 `distance = embedding <=> query`,再按 `distance <= 1-threshold` 过滤,`score = 1 - distance`。事务内 `SET LOCAL hnsw.ef_search`(≥40)与 `SET LOCAL hnsw.iterative_scan = strict_order`(pgvector ≥ 0.8,选择性过滤下持续补召回),老版本 GUC 不存在时自动降级重试。-- **关键词检索**:ParadeDB `pg_search` BM25——`content ||| query`(任意 token 匹配)+ `paradedb.score(id) as score`。+- **关键词检索**:`pg_search` BM25——`content ||| query`(任意 token 匹配)。首次关键词查询探测评分函数,优先使用 `paradedb.score(anyelement) -> real`,兼容部分 PolarDB 实例的 `pgsearch.score(anyelement) -> real`;仅缓存成功结果,探测失败可重试。缓存按检索仓库实例隔离,纯向量查询不触发探测。+- **兼容边界**:评分函数适配不会安装扩展、创建包装函数或修改数据库结构;仍须预先具备兼容的 `pg_search`、`|||` 操作符及有效 BM25 索引。不代表所有 PolarDB 版本均受支持;没有匹配评分函数时,关键词检索返回明确错误。- **过滤**:`knowledge_base_id` / `knowledge_id` / `tag_id` IN 过滤(AND 语义),`is_enabled` 为 NULL 或 true。- **建索引**:`BatchSave` + `ON CONFLICT DO NOTHING`;删除按 chunk/source/knowledge ID 物理删除。