Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(postgres): support pgsearch keyword scoring (#4502)

bert pan9110cb2da170
diff --git a/internal/application/repository/retriever/postgres/keyword_score_integration_test.go b/internal/application/repository/retriever/postgres/keyword_score_integration_test.go
new file mode 100644
index 000000000..666bf7132
--- /dev/null
+++ b/internal/application/repository/retriever/postgres/keyword_score_integration_test.go
@@ -0,0 +1,104 @@
+package postgres
+
+import (
+ "context"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ gormpostgres "gorm.io/driver/postgres"
+ "gorm.io/gorm"
+ "gorm.io/gorm/logger"
+)
+
+// TestKeywordScoreCatalogCompatibility checks the actual PostgreSQL catalog
+// query, including preference and signatures, rather than mocking its result.
+// Set WEKNORA_RETRIEVER_TEST_POSTGRES_DSN to a disposable PostgreSQL database
+// without pg_search installed. Fixtures are rolled back after each subtest;
+// existing scoring schemas are never replaced or dropped.
+func TestKeywordScoreCatalogCompatibility(t *testing.T) {
+ dsn := os.Getenv("WEKNORA_RETRIEVER_TEST_POSTGRES_DSN")
+ if dsn == "" {
+ t.Skip("set WEKNORA_RETRIEVER_TEST_POSTGRES_DSN to run PostgreSQL catalog tests")
+ }
+ db, err := gorm.Open(gormpostgres.Open(dsn), &gorm.Config{
+ Logger: logger.Default.LogMode(logger.Silent),
+ })
+ require.NoError(t, err)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = sqlDB.Close() })
+
+ var existing int
+ require.NoError(t, db.Raw(`SELECT count(*) FROM pg_catalog.pg_namespace
+ WHERE nspname IN ('paradedb', 'pgsearch', 'unrelated_score_test')`).Scan(&existing).Error)
+ require.Zero(t, existing, "use a disposable database without existing scoring schemas")
+
+ paradedb := "CREATE FUNCTION paradedb.score(anyelement) RETURNS real LANGUAGE sql AS 'SELECT 1::real'"
+ pgsearch := "CREATE FUNCTION pgsearch.score(anyelement) RETURNS real LANGUAGE sql AS 'SELECT 1::real'"
+ for _, tc := range []struct {
+ name string
+ functions []string
+ want string
+ }{
+ {name: "ParadeDB", functions: []string{paradedb}, want: "paradedb.score"},
+ {name: "PolarDB_layout", functions: []string{pgsearch}, want: "pgsearch.score"},
+ {name: "prefer_ParadeDB", functions: []string{pgsearch, paradedb}, want: "paradedb.score"},
+ {name: "missing_functions"},
+ {
+ name: "incompatible_argument_falls_back",
+ functions: []string{
+ "CREATE FUNCTION paradedb.score(integer) RETURNS real LANGUAGE sql AS 'SELECT 1::real'",
+ pgsearch,
+ },
+ want: "pgsearch.score",
+ },
+ {
+ name: "incompatible_result_falls_back",
+ functions: []string{
+ "CREATE FUNCTION paradedb.score(anyelement) RETURNS double precision LANGUAGE sql AS 'SELECT 1::real'",
+ pgsearch,
+ },
+ want: "pgsearch.score",
+ },
+ {
+ name: "reject_wrong_arity",
+ functions: []string{
+ "CREATE FUNCTION pgsearch.score(anyelement, integer) RETURNS real LANGUAGE sql AS 'SELECT 1::real'",
+ },
+ },
+ {
+ name: "reject_unlisted_schema",
+ functions: []string{
+ "CREATE FUNCTION unrelated_score_test.score(anyelement) RETURNS real LANGUAGE sql AS 'SELECT 1::real'",
+ },
+ },
+ {
+ name: "reject_wrong_result",
+ functions: []string{
+ "CREATE FUNCTION pgsearch.score(anyelement) RETURNS double precision LANGUAGE sql AS 'SELECT 1::real'",
+ },
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ tx := db.Begin()
+ require.NoError(t, tx.Error)
+ t.Cleanup(func() { require.NoError(t, tx.Rollback().Error) })
+ for _, schema := range []string{"paradedb", "pgsearch", "unrelated_score_test"} {
+ require.NoError(t, tx.Exec("CREATE SCHEMA "+schema).Error)
+ }
+ for _, ddl := range tc.functions {
+ require.NoError(t, tx.Exec(ddl).Error)
+ }
+ repo := &pgRepository{db: tx}
+ function, err := repo.resolveKeywordScoreFunction(context.Background())
+ if tc.want == "" {
+ require.ErrorContains(t, err, "check pg_search extension compatibility")
+ require.Empty(t, function)
+ return
+ }
+ require.NoError(t, err)
+ require.Equal(t, tc.want, function)
+ })
+ }
+}
diff --git a/internal/application/repository/retriever/postgres/keyword_score_test.go b/internal/application/repository/retriever/postgres/keyword_score_test.go
new file mode 100644
index 000000000..22b0564d0
--- /dev/null
+++ b/internal/application/repository/retriever/postgres/keyword_score_test.go
@@ -0,0 +1,193 @@
+package postgres
+
+import (
+ "context"
+ "errors"
+ "regexp"
+ "sync"
+ "testing"
+
+ "github.com/DATA-DOG/go-sqlmock"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+ gormpostgres "gorm.io/driver/postgres"
+ "gorm.io/gorm"
+ "gorm.io/gorm/logger"
+)
+
+func newKeywordTestRepository(t *testing.T) (*pgRepository, sqlmock.Sqlmock) {
+ t.Helper()
+ db, mock, err := sqlmock.New()
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, mock.ExpectationsWereMet()); _ = db.Close() })
+ gdb, err := gorm.Open(gormpostgres.New(gormpostgres.Config{Conn: db}), &gorm.Config{
+ DisableAutomaticPing: true,
+ Logger: logger.Default.LogMode(logger.Silent),
+ })
+ require.NoError(t, err)
+ return NewPostgresRetrieveEngineRepository(gdb).(*pgRepository), mock
+}
+
+// Exercise the actual query path for both backends, including scope filters,
+// bound query text, score mapping, and reuse of the per-connection detection.
+func TestKeywordsRetrieveScoreFunctionCompatibility(t *testing.T) {
+ for _, scoreFunction := range []string{"paradedb.score", "pgsearch.score"} {
+ t.Run(scoreFunction, func(t *testing.T) {
+ repo, mock := newKeywordTestRepository(t)
+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).
+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow(scoreFunction))
+ params := types.RetrieveParams{
+ RetrieverType: types.KeywordsRetrieverType,
+ Query: "关键词测试 ' OR 1=1 --", TopK: 5,
+ KnowledgeBaseIDs: []string{"kb-test"}, KnowledgeIDs: []string{"doc-test"},
+ TagIDs: []string{"tag-test"},
+ }
+ query := `SELECT ` + regexp.QuoteMeta(scoreFunction) + `\(id\) as score,.*` +
+ `FROM "embeddings" WHERE "knowledge_base_id" = \$1.*` +
+ `"knowledge_id" = \$2.*"tag_id" = \$3.*` +
+ `content \|\|\| \$4.*is_enabled IS NULL OR is_enabled = \$5.*` +
+ `ORDER BY "score" DESC LIMIT \$6`
+ for range 2 {
+ mock.ExpectQuery(query).
+ WithArgs("kb-test", "doc-test", "tag-test", params.Query, true, 5).
+ WillReturnRows(sqlmock.NewRows([]string{
+ "id", "chunk_id", "knowledge_id", "knowledge_base_id", "tag_id", "score",
+ }).
+ AddRow(1, "chunk-test", "doc-test", "kb-test", "tag-test", 31.15))
+ results, err := repo.Retrieve(context.Background(), params)
+ require.NoError(t, err)
+ require.Len(t, results, 1)
+ require.Len(t, results[0].Results, 1)
+ require.Equal(t, "chunk-test", results[0].Results[0].ChunkID)
+ require.Equal(t, 31.15, results[0].Results[0].Score)
+ require.Equal(t, types.MatchTypeKeywords, results[0].Results[0].MatchType)
+ }
+ })
+ }
+}
+
+func TestKeywordScoreDetectionFailuresDoNotPoisonCache(t *testing.T) {
+ for _, kind := range []string{"missing", "unexpected", "database_error"} {
+ t.Run(kind, func(t *testing.T) {
+ repo, mock := newKeywordTestRepository(t)
+ probe := mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL))
+ cause := errors.New("catalog unavailable")
+ switch kind {
+ case "missing":
+ probe.WillReturnRows(sqlmock.NewRows([]string{"score_function"}))
+ case "unexpected":
+ probe.WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("untrusted.score"))
+ default:
+ probe.WillReturnError(cause)
+ }
+ results, err := repo.KeywordsRetrieve(context.Background(), types.RetrieveParams{Query: "关键词测试", TopK: 5})
+ require.Nil(t, results)
+ if kind == "database_error" {
+ require.ErrorIs(t, err, cause)
+ } else {
+ require.ErrorContains(t, err, "check pg_search extension compatibility")
+ }
+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).
+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("pgsearch.score"))
+ function, err := repo.resolveKeywordScoreFunction(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "pgsearch.score", function)
+ })
+ }
+}
+
+func TestKeywordScoreDetectionConcurrentAndPerRepository(t *testing.T) {
+ repo, mock := newKeywordTestRepository(t)
+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).
+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("paradedb.score"))
+ var wg sync.WaitGroup
+ for range 16 {
+ wg.Go(func() {
+ function, err := repo.resolveKeywordScoreFunction(context.Background())
+ if err != nil || function != "paradedb.score" {
+ t.Errorf("resolve score function = %q, %v", function, err)
+ }
+ })
+ }
+ wg.Wait()
+ other, otherMock := newKeywordTestRepository(t)
+ otherMock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).
+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("pgsearch.score"))
+ function, err := other.resolveKeywordScoreFunction(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "pgsearch.score", function)
+}
+
+func TestKeywordScoreDetectionCanceledContextCanRetry(t *testing.T) {
+ repo, mock := newKeywordTestRepository(t)
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ // A canceled caller must not cache a failed detection or block later calls.
+ _, err := repo.resolveKeywordScoreFunction(ctx)
+ require.ErrorIs(t, err, context.Canceled)
+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).
+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("paradedb.score"))
+ function, err := repo.resolveKeywordScoreFunction(context.Background())
+ require.NoError(t, err)
+ require.Equal(t, "paradedb.score", function)
+}
+
+func TestKeywordsRetrieveQueryFailurePreservesResolvedFunction(t *testing.T) {
+ repo, mock := newKeywordTestRepository(t)
+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).
+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}).AddRow("pgsearch.score"))
+ query := `SELECT pgsearch\.score\(id\) as score,.*FROM "embeddings".*`
+ cause := errors.New("keyword index unavailable")
+ mock.ExpectQuery(query).WithArgs("关键词测试", true, 5).WillReturnError(cause)
+ params := types.RetrieveParams{Query: "关键词测试", TopK: 5}
+ results, err := repo.KeywordsRetrieve(context.Background(), params)
+ require.Nil(t, results)
+ require.ErrorIs(t, err, cause)
+
+ // The catalog probe succeeded: retry only the retrieval, not detection.
+ mock.ExpectQuery(query).WithArgs("关键词测试", true, 5).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "score"}))
+ results, err = repo.KeywordsRetrieve(context.Background(), params)
+ require.NoError(t, err)
+ require.Len(t, results, 1)
+ require.Empty(t, results[0].Results)
+}
+
+func TestVectorRetrieveDoesNotRequireKeywordScoreFunction(t *testing.T) {
+ for _, keywordFailed := range []bool{false, true} {
+ name := "without_keyword_probe"
+ if keywordFailed {
+ name = "after_unsupported_keyword_probe"
+ }
+ t.Run(name, func(t *testing.T) {
+ repo, mock := newKeywordTestRepository(t)
+ if keywordFailed {
+ mock.ExpectQuery(regexp.QuoteMeta(keywordScoreFunctionSQL)).
+ WillReturnRows(sqlmock.NewRows([]string{"score_function"}))
+ _, err := repo.KeywordsRetrieve(context.Background(), types.RetrieveParams{Query: "关键词测试", TopK: 5})
+ require.Error(t, err)
+ }
+
+ // Expect only vector SQL: any score-function probe fails this test.
+ mock.ExpectBegin()
+ mock.ExpectExec("SET LOCAL hnsw.ef_search = 100").WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectExec("SET LOCAL hnsw.iterative_scan = strict_order").WillReturnResult(sqlmock.NewResult(0, 0))
+ mock.ExpectQuery(`SELECT .*FROM embeddings.*`).
+ WithArgs(sqlmock.AnyArg(), 2, true, 100, 0.5, 3).
+ WillReturnRows(sqlmock.NewRows([]string{"id", "chunk_id", "score"}).AddRow(1, "vector-chunk", 0.9))
+ mock.ExpectCommit()
+ results, err := repo.Retrieve(context.Background(), types.RetrieveParams{
+ RetrieverType: types.VectorRetrieverType,
+ Embedding: []float32{0.1, 0.2},
+ Threshold: 0.5,
+ TopK: 3,
+ })
+ require.NoError(t, err)
+ require.Len(t, results, 1)
+ require.Len(t, results[0].Results, 1)
+ require.Equal(t, "vector-chunk", results[0].Results[0].ChunkID)
+ require.Equal(t, types.MatchTypeEmbedding, results[0].Results[0].MatchType)
+ })
+ }
+}
diff --git a/internal/application/repository/retriever/postgres/repository.go b/internal/application/repository/retriever/postgres/repository.go
index 4d1829ec7..cf28a9a27 100644
--- a/internal/application/repository/retriever/postgres/repository.go
+++ b/internal/application/repository/retriever/postgres/repository.go
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"strings"
+ "sync"
"github.com/Tencent/WeKnora/internal/common"
"github.com/Tencent/WeKnora/internal/logger"
@@ -18,7 +19,9 @@ import (
// pgRepository implements PostgreSQL-based retrieval operations
type pgRepository struct {
- db *gorm.DB // Database connection
+ db *gorm.DB // Database connection
+ keywordScoreMu sync.Mutex
+ keywordScoreFunction string
}
// NewPostgresRetrieveEngineRepository creates a new PostgreSQL retriever repository
@@ -27,6 +30,38 @@ func NewPostgresRetrieveEngineRepository(db *gorm.DB) interfaces.RetrieveEngineR
return &pgRepository{db: db}
}
+// keywordScoreFunctionSQL only selects fixed, schema-qualified function names.
+// Some PolarDB pg_search builds expose pgsearch.score instead of paradedb.score.
+const keywordScoreFunctionSQL = `SELECT n.nspname || '.score' AS score_function
+FROM pg_catalog.pg_proc p
+JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+WHERE n.nspname IN ('paradedb', 'pgsearch')
+ AND p.proname = 'score' AND p.prokind = 'f' AND p.pronargs = 1
+ AND p.proargtypes[0] = 'pg_catalog.anyelement'::regtype
+ AND p.prorettype = 'pg_catalog.float4'::regtype
+ORDER BY CASE n.nspname WHEN 'paradedb' THEN 0 ELSE 1 END
+LIMIT 1`
+
+// resolveKeywordScoreFunction probes lazily so vector-only stores remain usable.
+// Cache successes per repository/database; transient failures can be retried.
+func (g *pgRepository) resolveKeywordScoreFunction(ctx context.Context) (string, error) {
+ g.keywordScoreMu.Lock()
+ defer g.keywordScoreMu.Unlock()
+ if g.keywordScoreFunction != "" {
+ return g.keywordScoreFunction, nil
+ }
+ var row struct{ ScoreFunction string }
+ if err := g.db.WithContext(ctx).Raw(keywordScoreFunctionSQL).Scan(&row).Error; err != nil {
+ return "", fmt.Errorf("detect PostgreSQL keyword score function: %w", err)
+ }
+ if row.ScoreFunction != "paradedb.score" && row.ScoreFunction != "pgsearch.score" {
+ return "", errors.New("PostgreSQL keyword retrieval requires paradedb.score(anyelement) or " +
+ "pgsearch.score(anyelement); check pg_search extension compatibility")
+ }
+ g.keywordScoreFunction = row.ScoreFunction
+ return g.keywordScoreFunction, nil
+}
+
// EngineType returns the retriever engine type (PostgreSQL)
func (r *pgRepository) EngineType() types.RetrieverEngineType {
return types.PostgresRetrieverEngineType
@@ -191,6 +226,10 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
params types.RetrieveParams,
) ([]*types.RetrieveResult, error) {
logger.GetLogger(ctx).Infof("[Postgres] Keywords retrieval: query=%s, topK=%d", params.Query, params.TopK)
+ scoreFunction, err := g.resolveKeywordScoreFunction(ctx)
+ if err != nil {
+ return nil, err
+ }
conds := make([]clause.Expression, 0)
// KnowledgeBaseIDs and KnowledgeIDs use AND logic
@@ -236,9 +275,9 @@ func (g *pgRepository) KeywordsRetrieve(ctx context.Context,
}})
var embeddingDBList []pgVectorWithScore
- err := g.db.WithContext(ctx).Clauses(conds...).
+ err = g.db.WithContext(ctx).Clauses(conds...).
Select([]string{
- "paradedb.score(id) as score",
+ scoreFunction + "(id) as score",
"id",
"content",
"source_id",
diff --git a/website-docs/03-features/05-retrieval-engines.md b/website-docs/03-features/05-retrieval-engines.md
index d544bc483..7286faea5 100644
--- a/website-docs/03-features/05-retrieval-engines.md
+++ b/website-docs/03-features/05-retrieval-engines.md
@@ -23,7 +23,7 @@
| 引擎 | RETRIEVE_DRIVER 值 | 向量检索 | 关键词/全文 | 关键词打分 | 中文分词 | 维度管理 | 阈值下推 | 部署复杂度 | 适用场景 |
|------|-------------------|----------|------------|-----------|---------|----------|---------|-----------|----------|
-| PostgreSQL | `postgres` | pgvector halfvec + HNSW 表达式索引 | ParadeDB BM25(`\|\|\|`) | BM25(paradedb.score) | ParadeDB tokenizer | 单表混维,表达式索引按维 cast | 距离阈值 SQL 内 | 低(默认镜像内置) | 默认选择;与业务同库,事务一致 |
+| PostgreSQL | `postgres` | pgvector halfvec + HNSW 表达式索引 | pg_search BM25(`\|\|\|`) | BM25(优先 `paradedb.score`,兼容 `pgsearch.score`) | pg_search tokenizer | 单表混维,表达式索引按维 cast | 距离阈值 SQL 内 | 低(默认镜像内置) | 默认选择;与业务同库,事务一致 |
| SQLite | `sqlite` | sqlite-vec vec0(cosine) | FTS5 contentless | FTS5 | 应用侧 bigram | 每维一张 vec0 虚表 | 应用侧 | 极低(内嵌) | 桌面版 / 开发 / 微型部署 |
| Elasticsearch v8 | `elasticsearch_v8` | script_score cosineSimilarity | match(BM25) | BM25 | ES analyzer | dense_vector 单索引 | 应用侧 | 中 | 已有 ES 8 集群 |
| Elasticsearch v7 | `elasticsearch_v7` | 不支持(Support 仅 keywords) | match(BM25) | BM25 | ES analyzer | — | — | 中 | 存量 ES 7,仅作关键词引擎,需与其他向量引擎组合 |
@@ -125,7 +125,8 @@ flowchart TD
`internal/application/repository/retriever/postgres/repository.go`。数据与业务库同库(`embeddings` 表,GORM 管理)。
- **向量检索**:pgvector `halfvec`(半精度,2 字节/维)。`embedding` 列不定维,HNSW 索引建在表达式 `(embedding::halfvec(dim)) halfvec_cosine_ops` 上——**ORDER BY 表达式必须与索引表达式完全一致**(两侧显式 cast),否则退化为顺序扫描(源码注释引 pgvector issue [#702](https://github.com/pgvector/pgvector/issues/702)/[#835](https://github.com/pgvector/pgvector/issues/835))。查询用子查询先取 `expandedTopK`(TopK*2,夹在 [100,200],避免大 LIMIT 拖垮 HNSW)个候选算 `distance = embedding <=> query`,再按 `distance <= 1-threshold` 过滤,`score = 1 - distance`。事务内 `SET LOCAL hnsw.ef_search`(≥40)与 `SET LOCAL hnsw.iterative_scan = strict_order`(pgvector ≥ 0.8,选择性过滤下持续补召回),老版本 GUC 不存在时自动降级重试。
-- **关键词检索**:ParadeDB `pg_search` BM25——`content ||| query`(任意 token 匹配)+ `paradedb.score(id) as score`。
+- **关键词检索**:`pg_search` BM25——`content ||| query`(任意 token 匹配)。首次关键词查询探测评分函数,优先使用 `paradedb.score(anyelement) -> real`,兼容部分 PolarDB 实例的 `pgsearch.score(anyelement) -> real`;仅缓存成功结果,探测失败可重试。缓存按检索仓库实例隔离,纯向量查询不触发探测。
+- **兼容边界**:评分函数适配不会安装扩展、创建包装函数或修改数据库结构;仍须预先具备兼容的 `pg_search`、`|||` 操作符及有效 BM25 索引。不代表所有 PolarDB 版本均受支持;没有匹配评分函数时,关键词检索返回明确错误。
- **过滤**:`knowledge_base_id` / `knowledge_id` / `tag_id` IN 过滤(AND 语义),`is_enabled` 为 NULL 或 true。
- **建索引**:`BatchSave` + `ON CONFLICT DO NOTHING`;删除按 chunk/source/knowledge ID 物理删除。