Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

fix(database/knowledge): replace SimilaritySearch with direct SQLC vector queries

Dmitry Ng0f0a7bd2d0ab
diff --git a/backend/pkg/database/knowledge.sql.go b/backend/pkg/database/knowledge.sql.go
index ddbfb7bf..b6c6c8db 100644
--- a/backend/pkg/database/knowledge.sql.go
+++ b/backend/pkg/database/knowledge.sql.go
@@ -21,10 +21,10 @@ WHERE collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'la
// Delete all memory-type documents for a specific flow.
// Called on flow deletion to free long-term memory that will never be re-used.
-// $1 is the decimal text representation of the flow ID (e.g. "55"), matching the
+// flow_id is the decimal text representation of the flow ID (e.g. "55"), matching the
// text result of (cmetadata ->> 'flow_id') which uses JSON ->> extraction.
-func (q *Queries) DeleteFlowMemoryDocuments(ctx context.Context, dollar_1 sql.NullString) error {
- _, err := q.db.ExecContext(ctx, deleteFlowMemoryDocuments, dollar_1)
+func (q *Queries) DeleteFlowMemoryDocuments(ctx context.Context, flowID sql.NullString) error {
+ _, err := q.db.ExecContext(ctx, deleteFlowMemoryDocuments, flowID)
return err
}
@@ -35,8 +35,8 @@ WHERE uuid::text = $1
`
// Delete a knowledge document by UUID (admin — no user_id check).
-func (q *Queries) DeleteKnowledgeDocument(ctx context.Context, dollar_1 sql.NullString) error {
- _, err := q.db.ExecContext(ctx, deleteKnowledgeDocument, dollar_1)
+func (q *Queries) DeleteKnowledgeDocument(ctx context.Context, uuid sql.NullString) error {
+ _, err := q.db.ExecContext(ctx, deleteKnowledgeDocument, uuid)
return err
}
@@ -48,13 +48,13 @@ WHERE uuid::text = $1
`
type DeleteUserKnowledgeDocumentParams struct {
- Column1 sql.NullString `json:"column_1"`
- Column2 sql.NullString `json:"column_2"`
+ Uuid sql.NullString `json:"uuid"`
+ UserID sql.NullString `json:"user_id"`
}
// Delete a knowledge document by UUID, only if it belongs to the given user.
func (q *Queries) DeleteUserKnowledgeDocument(ctx context.Context, arg DeleteUserKnowledgeDocumentParams) error {
- _, err := q.db.ExecContext(ctx, deleteUserKnowledgeDocument, arg.Column1, arg.Column2)
+ _, err := q.db.ExecContext(ctx, deleteUserKnowledgeDocument, arg.Uuid, arg.UserID)
return err
}
@@ -96,8 +96,8 @@ WHERE c.name = 'langchain'
`
type GetUserKnowledgeDocumentParams struct {
- Uuid string `json:"uuid"`
- Cmetadata sql.NullString `json:"cmetadata"`
+ Uuid string `json:"uuid"`
+ UserID sql.NullString `json:"user_id"`
}
type GetUserKnowledgeDocumentRow struct {
@@ -108,7 +108,7 @@ type GetUserKnowledgeDocumentRow struct {
// Fetch a single knowledge document by UUID, scoped to a specific user.
func (q *Queries) GetUserKnowledgeDocument(ctx context.Context, arg GetUserKnowledgeDocumentParams) (GetUserKnowledgeDocumentRow, error) {
- row := q.db.QueryRowContext(ctx, getUserKnowledgeDocument, arg.Uuid, arg.Cmetadata)
+ row := q.db.QueryRowContext(ctx, getUserKnowledgeDocument, arg.Uuid, arg.UserID)
var i GetUserKnowledgeDocumentRow
err := row.Scan(&i.ID, &i.Document, &i.Cmetadata)
return i, err
@@ -176,8 +176,8 @@ type ListFlowKnowledgeDocumentsRow struct {
}
// List non-memory knowledge documents belonging to a specific flow (admin scoped).
-func (q *Queries) ListFlowKnowledgeDocuments(ctx context.Context, cmetadata sql.NullString) ([]ListFlowKnowledgeDocumentsRow, error) {
- rows, err := q.db.QueryContext(ctx, listFlowKnowledgeDocuments, cmetadata)
+func (q *Queries) ListFlowKnowledgeDocuments(ctx context.Context, flowID sql.NullString) ([]ListFlowKnowledgeDocumentsRow, error) {
+ rows, err := q.db.QueryContext(ctx, listFlowKnowledgeDocuments, flowID)
if err != nil {
return nil, err
}
@@ -219,8 +219,8 @@ type ListUserKnowledgeDocumentsRow struct {
}
// List all non-memory knowledge documents owned by a specific user (user-scoped view).
-func (q *Queries) ListUserKnowledgeDocuments(ctx context.Context, cmetadata sql.NullString) ([]ListUserKnowledgeDocumentsRow, error) {
- rows, err := q.db.QueryContext(ctx, listUserKnowledgeDocuments, cmetadata)
+func (q *Queries) ListUserKnowledgeDocuments(ctx context.Context, userID sql.NullString) ([]ListUserKnowledgeDocumentsRow, error) {
+ rows, err := q.db.QueryContext(ctx, listUserKnowledgeDocuments, userID)
if err != nil {
return nil, err
}
@@ -242,13 +242,146 @@ func (q *Queries) ListUserKnowledgeDocuments(ctx context.Context, cmetadata sql.
return items, nil
}
+const searchKnowledgeDocuments = `-- name: SearchKnowledgeDocuments :many
+SELECT
+ e.uuid::text AS id,
+ COALESCE(e.document, '') AS document,
+ COALESCE(e.cmetadata::text, '{}') AS cmetadata,
+ (1.0 - (e.embedding <=> $1::vector))::float8 AS score
+FROM langchain_pg_embedding e
+INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid
+WHERE c.name = 'langchain'
+ AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')
+ AND (e.embedding <=> $1::vector)::float8 < $2::float8
+ORDER BY e.embedding <=> $1::vector
+LIMIT $3::int
+`
+
+type SearchKnowledgeDocumentsParams struct {
+ Embedding interface{} `json:"embedding"`
+ MaxDistance float64 `json:"max_distance"`
+ Lim int32 `json:"lim"`
+}
+
+type SearchKnowledgeDocumentsRow struct {
+ ID string `json:"id"`
+ Document string `json:"document"`
+ Cmetadata sql.NullString `json:"cmetadata"`
+ Score float64 `json:"score"`
+}
+
+// Vector similarity search over all knowledge documents (admin view, no user filter).
+// Returns rows ordered by cosine similarity descending (highest score first).
+// embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'
+// max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold),
+//
+// e.g. pass 0.8 to get documents with similarity score > 0.2
+//
+// lim maximum number of rows to return
+func (q *Queries) SearchKnowledgeDocuments(ctx context.Context, arg SearchKnowledgeDocumentsParams) ([]SearchKnowledgeDocumentsRow, error) {
+ rows, err := q.db.QueryContext(ctx, searchKnowledgeDocuments, arg.Embedding, arg.MaxDistance, arg.Lim)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ var items []SearchKnowledgeDocumentsRow
+ for rows.Next() {
+ var i SearchKnowledgeDocumentsRow
+ if err := rows.Scan(
+ &i.ID,
+ &i.Document,
+ &i.Cmetadata,
+ &i.Score,
+ ); err != nil {
+ return nil, err
+ }
+ items = append(items, i)
+ }
+ if err := rows.Close(); err != nil {
+ return nil, err
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
+
+const searchUserKnowledgeDocuments = `-- name: SearchUserKnowledgeDocuments :many
+SELECT
+ e.uuid::text AS id,
+ COALESCE(e.document, '') AS document,
+ COALESCE(e.cmetadata::text, '{}') AS cmetadata,
+ (1.0 - (e.embedding <=> $1::vector))::float8 AS score
+FROM langchain_pg_embedding e
+INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid
+WHERE c.name = 'langchain'
+ AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')
+ AND (e.cmetadata ->> 'user_id') = $2
+ AND (e.embedding <=> $1::vector)::float8 < $3::float8
+ORDER BY e.embedding <=> $1::vector
+LIMIT $4::int
+`
+
+type SearchUserKnowledgeDocumentsParams struct {
+ Embedding interface{} `json:"embedding"`
+ UserID sql.NullString `json:"user_id"`
+ MaxDistance float64 `json:"max_distance"`
+ Lim int32 `json:"lim"`
+}
+
+type SearchUserKnowledgeDocumentsRow struct {
+ ID string `json:"id"`
+ Document string `json:"document"`
+ Cmetadata sql.NullString `json:"cmetadata"`
+ Score float64 `json:"score"`
+}
+
+// Vector similarity search scoped to a specific user (by cmetadata user_id).
+// Returns rows ordered by cosine similarity descending (highest score first).
+// embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'
+// max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold)
+// lim maximum number of rows to return
+// user_id owner filter as a decimal text string (e.g. "42")
+func (q *Queries) SearchUserKnowledgeDocuments(ctx context.Context, arg SearchUserKnowledgeDocumentsParams) ([]SearchUserKnowledgeDocumentsRow, error) {
+ rows, err := q.db.QueryContext(ctx, searchUserKnowledgeDocuments,
+ arg.Embedding,
+ arg.UserID,
+ arg.MaxDistance,
+ arg.Lim,
+ )
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ var items []SearchUserKnowledgeDocumentsRow
+ for rows.Next() {
+ var i SearchUserKnowledgeDocumentsRow
+ if err := rows.Scan(
+ &i.ID,
+ &i.Document,
+ &i.Cmetadata,
+ &i.Score,
+ ); err != nil {
+ return nil, err
+ }
+ items = append(items, i)
+ }
+ if err := rows.Close(); err != nil {
+ return nil, err
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return items, nil
+}
+
const updateKnowledgeDocument = `-- name: UpdateKnowledgeDocument :one
UPDATE langchain_pg_embedding
SET
- embedding = $2::vector,
- document = $3,
- cmetadata = $4::json
-WHERE uuid::text = $1
+ embedding = $1::vector,
+ document = $2,
+ cmetadata = $3::json
+WHERE uuid::text = $4
AND collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain')
RETURNING
uuid::text AS id,
@@ -257,10 +390,10 @@ RETURNING
`
type UpdateKnowledgeDocumentParams struct {
- Column1 sql.NullString `json:"column_1"`
- Column2 string `json:"column_2"`
- Column3 sql.NullString `json:"column_3"`
- Column4 pqtype.NullRawMessage `json:"column_4"`
+ Embedding string `json:"embedding"`
+ Document sql.NullString `json:"document"`
+ Cmetadata pqtype.NullRawMessage `json:"cmetadata"`
+ Uuid sql.NullString `json:"uuid"`
}
type UpdateKnowledgeDocumentRow struct {
@@ -270,14 +403,14 @@ type UpdateKnowledgeDocumentRow struct {
}
// Update an existing document's embedding, text and metadata atomically.
-// $2 must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'
-// $4 must be valid JSON text.
+// embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'
+// cmetadata must be valid JSON text.
func (q *Queries) UpdateKnowledgeDocument(ctx context.Context, arg UpdateKnowledgeDocumentParams) (UpdateKnowledgeDocumentRow, error) {
row := q.db.QueryRowContext(ctx, updateKnowledgeDocument,
- arg.Column1,
- arg.Column2,
- arg.Column3,
- arg.Column4,
+ arg.Embedding,
+ arg.Document,
+ arg.Cmetadata,
+ arg.Uuid,
)
var i UpdateKnowledgeDocumentRow
err := row.Scan(&i.ID, &i.Document, &i.Cmetadata)
diff --git a/backend/pkg/database/knowledge/knowledge.go b/backend/pkg/database/knowledge/knowledge.go
index f6dcd8b7..794210a8 100644
--- a/backend/pkg/database/knowledge/knowledge.go
+++ b/backend/pkg/database/knowledge/knowledge.go
@@ -20,7 +20,6 @@ import (
)
const (
- collectionName = "langchain"
defaultSearchLimit = 10
defaultSearchThreshold = float32(0.2)
manualDocumentFlag = "manual"
@@ -224,34 +223,6 @@ func applyGoFilters(docs []*model.KnowledgeDocument, filter *model.KnowledgeFilt
return result
}
-// buildSearchFilters converts a KnowledgeFilter to a pgvector filter map.
-// Only single-value equality filters are expressed here; arrays are applied in Go.
-func buildSearchFilters(userID int64, filter *model.KnowledgeFilter) map[string]any {
- f := make(map[string]any)
- if userID > 0 {
- f["user_id"] = strconv.FormatInt(userID, 10)
- }
- if filter == nil {
- return f
- }
- if len(filter.DocTypes) == 1 {
- f["doc_type"] = string(filter.DocTypes[0])
- }
- if len(filter.GuideTypes) == 1 {
- f["guide_type"] = string(filter.GuideTypes[0])
- }
- if len(filter.AnswerTypes) == 1 {
- f["answer_type"] = string(filter.AnswerTypes[0])
- }
- if len(filter.CodeLangs) == 1 {
- f["code_lang"] = filter.CodeLangs[0]
- }
- if filter.FlowID != nil {
- f["flow_id"] = strconv.FormatInt(*filter.FlowID, 10)
- }
- return f
-}
-
func (ks *knowledgeStore) requireStore() error {
if ks.store == nil {
return fmt.Errorf("knowledge: embedding provider is not configured")
@@ -338,8 +309,8 @@ func (ks *knowledgeStore) GetDocument(ctx context.Context, id string) (*model.Kn
func (ks *knowledgeStore) GetUserDocument(ctx context.Context, userID int64, id string) (*model.KnowledgeDocument, error) {
row, err := ks.db.GetUserKnowledgeDocument(ctx, database.GetUserKnowledgeDocumentParams{
- Uuid: id,
- Cmetadata: nsOf(strconv.FormatInt(userID, 10)),
+ Uuid: id,
+ UserID: nsOf(strconv.FormatInt(userID, 10)),
})
if err != nil {
return nil, fmt.Errorf("knowledge: get user document %s: %w", id, err)
@@ -359,61 +330,108 @@ func (ks *knowledgeStore) SearchUserDocuments(ctx context.Context, userID int64,
return ks.doSearch(ctx, userID, query, filter, limit)
}
+// doSearch performs a parameterised vector similarity search and returns
+// matched documents with their cosine-similarity scores and correct UUIDs.
+//
+// Filters are applied in two layers:
+// 1. SQL layer — user_id ownership (when userID > 0); cosine-distance
+// threshold; row limit. All values are bound as query parameters so
+// there is no risk of SQL injection.
+// 2. Go layer — multi-value enum filters (DocTypes, GuideTypes, etc.) and
+// the Manual flag that cannot be expressed as static SQL predicates.
func (ks *knowledgeStore) doSearch(ctx context.Context, userID int64, query string, filter *model.KnowledgeFilter, limit int) ([]*model.KnowledgeDocumentWithScore, error) {
- if err := ks.requireStore(); err != nil {
+ if err := ks.requireEmbedder(); err != nil {
return nil, err
}
if limit <= 0 {
limit = defaultSearchLimit
}
- pgFilters := buildSearchFilters(userID, filter)
- opts := []vectorstores.Option{
- vectorstores.WithScoreThreshold(defaultSearchThreshold),
+ // Compute query embedding.
+ vecs, err := ks.embedder.EmbedDocuments(ctx, []string{query})
+ if err != nil {
+ return nil, fmt.Errorf("knowledge: embed query: %w", err)
}
- if len(pgFilters) > 0 {
- opts = append(opts, vectorstores.WithFilters(pgFilters))
+ if len(vecs) == 0 {
+ return nil, fmt.Errorf("knowledge: embedder returned no vectors for query")
}
- schemaDocs, err := ks.store.SimilaritySearch(ctx, query, limit, opts...)
- if err != nil {
- return nil, fmt.Errorf("knowledge: similarity search: %w", err)
+ vecLiteral := formatVector(vecs[0])
+ // maxDist is the cosine-distance upper bound (exclusive):
+ // distance = 1 - similarity, so maxDist = 1 - threshold.
+ maxDist := float64(1.0 - defaultSearchThreshold)
+
+ type searchRow struct {
+ ID string
+ Document string
+ Cmetadata sql.NullString
+ Score float64
}
- results := make([]*model.KnowledgeDocumentWithScore, 0, len(schemaDocs))
- for _, sd := range schemaDocs {
- doc := schemaDocToModel(sd)
- if !passesGoFilter(doc, filter) {
+ var rawRows []searchRow
+
+ if userID > 0 {
+ rows, err := ks.db.SearchUserKnowledgeDocuments(ctx, database.SearchUserKnowledgeDocumentsParams{
+ Embedding: vecLiteral,
+ UserID: nsOf(strconv.FormatInt(userID, 10)),
+ MaxDistance: maxDist,
+ Lim: int32(limit), //nolint:gosec
+ })
+ if err != nil {
+ return nil, fmt.Errorf("knowledge: similarity search (user %d): %w", userID, err)
+ }
+ rawRows = make([]searchRow, len(rows))
+ for i, r := range rows {
+ rawRows[i] = searchRow{r.ID, r.Document, r.Cmetadata, r.Score}
+ }
+ } else {
+ rows, err := ks.db.SearchKnowledgeDocuments(ctx, database.SearchKnowledgeDocumentsParams{
+ Embedding: vecLiteral,
+ MaxDistance: maxDist,
+ Lim: int32(limit), //nolint:gosec
+ })
+ if err != nil {
+ return nil, fmt.Errorf("knowledge: similarity search (admin): %w", err)
+ }
+ rawRows = make([]searchRow, len(rows))
+ for i, r := range rows {
+ rawRows[i] = searchRow{r.ID, r.Document, r.Cmetadata, r.Score}
+ }
+ }
+
+ results := make([]*model.KnowledgeDocumentWithScore, 0, len(rawRows))
+ for _, r := range rawRows {
+ doc := rowToModel(r.ID, r.Document, nullStr(r.Cmetadata), true)
+ if !passesSearchFilter(doc, filter) {
continue
}
results = append(results, &model.KnowledgeDocumentWithScore{
- Score: float64(sd.Score),
+ Score: r.Score,
Document: doc,
})
}
return results, nil
}
-func schemaDocToModel(sd schema.Document) *model.KnowledgeDocument {
- raw, _ := json.Marshal(sd.Metadata)
- meta := parseMeta(string(raw))
- return metaToModelDoc("", sd.PageContent, meta)
-}
-
-func passesGoFilter(doc *model.KnowledgeDocument, filter *model.KnowledgeFilter) bool {
+// passesSearchFilter checks the Go-side filters that cannot be expressed as
+// static SQL predicates (FlowID equality, multi-value enums, and the Manual flag).
+func passesSearchFilter(doc *model.KnowledgeDocument, filter *model.KnowledgeFilter) bool {
if filter == nil {
return true
}
- if len(filter.DocTypes) > 1 && !slices.Contains(filter.DocTypes, doc.DocType) {
+ if filter.FlowID != nil && (doc.FlowID == nil || *doc.FlowID != *filter.FlowID) {
+ return false
+ }
+ if len(filter.DocTypes) > 0 && !slices.Contains(filter.DocTypes, doc.DocType) {
return false
}
- if len(filter.GuideTypes) > 1 && (doc.GuideType == nil || !slices.Contains(filter.GuideTypes, *doc.GuideType)) {
+ if len(filter.GuideTypes) > 0 && (doc.GuideType == nil || !slices.Contains(filter.GuideTypes, *doc.GuideType)) {
return false
}
- if len(filter.AnswerTypes) > 1 && (doc.AnswerType == nil || !slices.Contains(filter.AnswerTypes, *doc.AnswerType)) {
+ if len(filter.AnswerTypes) > 0 && (doc.AnswerType == nil || !slices.Contains(filter.AnswerTypes, *doc.AnswerType)) {
return false
}
- if len(filter.CodeLangs) > 1 && (doc.CodeLang == nil || !slices.Contains(filter.CodeLangs, *doc.CodeLang)) {
+ if len(filter.CodeLangs) > 0 && (doc.CodeLang == nil || !slices.Contains(filter.CodeLangs, *doc.CodeLang)) {
return false
}
if filter.Manual != nil && doc.Manual != *filter.Manual {
@@ -588,10 +606,10 @@ func (ks *knowledgeStore) doUpdate(ctx context.Context, userID int64, id string,
}
row, err := ks.db.UpdateKnowledgeDocument(ctx, database.UpdateKnowledgeDocumentParams{
- Column1: nsOf(id), // uuid::text = $1
- Column2: formatVector(vecs[0]), // embedding = $2::vector
- Column3: nsOf(content), // document = $3
- Column4: cmJSON, // cmetadata = $4::json
+ Uuid: nsOf(id),
+ Embedding: formatVector(vecs[0]),
+ Document: nsOf(content),
+ Cmetadata: cmJSON,
})
if err != nil {
return nil, fmt.Errorf("knowledge: update document %s: %w", id, err)
@@ -624,8 +642,8 @@ func (ks *knowledgeStore) DeleteUserDocument(ctx context.Context, userID int64,
return err
}
if err := ks.db.DeleteUserKnowledgeDocument(ctx, database.DeleteUserKnowledgeDocumentParams{
- Column1: nsOf(id),
- Column2: nsOf(strconv.FormatInt(userID, 10)),
+ Uuid: nsOf(id),
+ UserID: nsOf(strconv.FormatInt(userID, 10)),
}); err != nil {
return fmt.Errorf("knowledge: delete user document %s: %w", id, err)
}
diff --git a/backend/pkg/database/knowledge/knowledge_test.go b/backend/pkg/database/knowledge/knowledge_test.go
index 1ee42fa1..2e2e9c0f 100644
--- a/backend/pkg/database/knowledge/knowledge_test.go
+++ b/backend/pkg/database/knowledge/knowledge_test.go
@@ -33,8 +33,10 @@ type mockDB struct {
listFlow func(ctx context.Context, flowID sql.NullString) ([]database.ListFlowKnowledgeDocumentsRow, error)
listUser func(ctx context.Context, userID sql.NullString) ([]database.ListUserKnowledgeDocumentsRow, error)
updateKnowledge func(ctx context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error)
- deleteKnowledge func(ctx context.Context, dollar_1 sql.NullString) error
+ deleteKnowledge func(ctx context.Context, uuid sql.NullString) error
deleteUserKnowledge func(ctx context.Context, arg database.DeleteUserKnowledgeDocumentParams) error
+ searchKnowledge func(ctx context.Context, arg database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error)
+ searchUserKnowledge func(ctx context.Context, arg database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error)
}
func (m *mockDB) GetKnowledgeDocument(ctx context.Context, uuid string) (database.GetKnowledgeDocumentRow, error) {
@@ -55,12 +57,24 @@ func (m *mockDB) ListUserKnowledgeDocuments(ctx context.Context, cmetadata sql.N
func (m *mockDB) UpdateKnowledgeDocument(ctx context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {
return m.updateKnowledge(ctx, arg)
}
-func (m *mockDB) DeleteKnowledgeDocument(ctx context.Context, dollar_1 sql.NullString) error {
- return m.deleteKnowledge(ctx, dollar_1)
+func (m *mockDB) DeleteKnowledgeDocument(ctx context.Context, uuid sql.NullString) error {
+ return m.deleteKnowledge(ctx, uuid)
}
func (m *mockDB) DeleteUserKnowledgeDocument(ctx context.Context, arg database.DeleteUserKnowledgeDocumentParams) error {
return m.deleteUserKnowledge(ctx, arg)
}
+func (m *mockDB) SearchKnowledgeDocuments(ctx context.Context, arg database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {
+ if m.searchKnowledge != nil {
+ return m.searchKnowledge(ctx, arg)
+ }
+ return nil, nil
+}
+func (m *mockDB) SearchUserKnowledgeDocuments(ctx context.Context, arg database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {
+ if m.searchUserKnowledge != nil {
+ return m.searchUserKnowledge(ctx, arg)
+ }
+ return nil, nil
+}
// --- mockVectorStore --------------------------------------------------------
@@ -158,6 +172,24 @@ func makeListUserRow(id, document, cmetadata string) database.ListUserKnowledgeD
}
}
+func makeSearchRow(id, document, cmetadata string, score float64) database.SearchKnowledgeDocumentsRow {
+ return database.SearchKnowledgeDocumentsRow{
+ ID: id,
+ Document: document,
+ Cmetadata: sql.NullString{String: cmetadata, Valid: true},
+ Score: score,
+ }
+}
+
+func makeUserSearchRow(id, document, cmetadata string, score float64) database.SearchUserKnowledgeDocumentsRow {
+ return database.SearchUserKnowledgeDocumentsRow{
+ ID: id,
+ Document: document,
+ Cmetadata: sql.NullString{String: cmetadata, Valid: true},
+ Score: score,
+ }
+}
+
// newPublisherFactory returns a PublisherFactory backed by a single shared publisher.
func newPublisherFactory(pub *mockPublisher) PublisherFactory {
return func(userID int64) subscriptions.KnowledgePublisher {
@@ -531,83 +563,448 @@ func TestApplyGoFilters(t *testing.T) {
}
// ============================================================================
-// Tests: buildSearchFilters
+// Tests: passesSearchFilter
// ============================================================================
-func TestBuildSearchFilters(t *testing.T) {
- t.Run("userID=0 produces no user_id key", func(t *testing.T) {
- f := buildSearchFilters(0, nil)
- if _, ok := f["user_id"]; ok {
- t.Fatal("user_id should be absent when userID=0")
+func TestPassesSearchFilter(t *testing.T) {
+ flowID10 := int64(10)
+ guide := &model.KnowledgeDocument{
+ DocType: model.KnowledgeDocTypeGuide,
+ GuideType: ptr(model.KnowledgeGuideTypePentest),
+ FlowID: &flowID10,
+ }
+ answer := &model.KnowledgeDocument{
+ DocType: model.KnowledgeDocTypeAnswer,
+ AnswerType: ptr(model.KnowledgeAnswerTypeVulnerability),
+ Manual: true,
+ }
+ code := &model.KnowledgeDocument{
+ DocType: model.KnowledgeDocTypeCode,
+ CodeLang: ptr("python"),
+ }
+
+ t.Run("nil filter passes everything", func(t *testing.T) {
+ if !passesSearchFilter(guide, nil) {
+ t.Fatal("nil filter must pass all documents")
}
})
- t.Run("userID>0 adds user_id string", func(t *testing.T) {
- f := buildSearchFilters(42, nil)
- if f["user_id"] != "42" {
- t.Fatalf("expected user_id=42, got %v", f["user_id"])
+ t.Run("empty filter passes everything", func(t *testing.T) {
+ if !passesSearchFilter(answer, &model.KnowledgeFilter{}) {
+ t.Fatal("empty filter must pass all documents")
}
})
- t.Run("nil filter returns only userID-based key", func(t *testing.T) {
- f := buildSearchFilters(1, nil)
- if len(f) != 1 {
- t.Fatalf("want 1 key, got %d", len(f))
+ t.Run("matching docType passes", func(t *testing.T) {
+ f := &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide}}
+ if !passesSearchFilter(guide, f) {
+ t.Fatal("matching docType must pass")
}
})
- t.Run("single docType sets doc_type", func(t *testing.T) {
- f := buildSearchFilters(0, &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide}})
- if f["doc_type"] != "guide" {
- t.Fatalf("expected doc_type=guide, got %v", f["doc_type"])
+ t.Run("non-matching docType blocks", func(t *testing.T) {
+ f := &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeCode}}
+ if passesSearchFilter(guide, f) {
+ t.Fatal("non-matching docType must block")
}
})
- t.Run("multiple docTypes does NOT set doc_type (handled in Go)", func(t *testing.T) {
- f := buildSearchFilters(0, &model.KnowledgeFilter{
- DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide, model.KnowledgeDocTypeCode},
- })
- if _, ok := f["doc_type"]; ok {
- t.Fatal("doc_type should not be set for multiple types")
+ t.Run("multiple docTypes: match passes", func(t *testing.T) {
+ f := &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide, model.KnowledgeDocTypeCode}}
+ if !passesSearchFilter(guide, f) {
+ t.Fatal("doc matching one of multiple types must pass")
}
})
- t.Run("single guideType sets guide_type", func(t *testing.T) {
- f := buildSearchFilters(0, &model.KnowledgeFilter{GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypePentest}})
- if f["guide_type"] != "pentest" {
- t.Fatalf("expected guide_type=pentest, got %v", f["guide_type"])
+ t.Run("matching guideType passes", func(t *testing.T) {
+ f := &model.KnowledgeFilter{GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypePentest}}
+ if !passesSearchFilter(guide, f) {
+ t.Fatal("matching guideType must pass")
}
})
- t.Run("single answerType sets answer_type", func(t *testing.T) {
- f := buildSearchFilters(0, &model.KnowledgeFilter{AnswerTypes: []model.KnowledgeAnswerType{model.KnowledgeAnswerTypeCode}})
- if f["answer_type"] != "code" {
- t.Fatalf("expected answer_type=code, got %v", f["answer_type"])
+ t.Run("non-matching guideType blocks", func(t *testing.T) {
+ f := &model.KnowledgeFilter{GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypeInstall}}
+ if passesSearchFilter(guide, f) {
+ t.Fatal("non-matching guideType must block")
}
})
- t.Run("single codeLang sets code_lang", func(t *testing.T) {
- f := buildSearchFilters(0, &model.KnowledgeFilter{CodeLangs: []string{"go"}})
- if f["code_lang"] != "go" {
- t.Fatalf("expected code_lang=go, got %v", f["code_lang"])
+ t.Run("doc without guideType is blocked by guideType filter", func(t *testing.T) {
+ f := &model.KnowledgeFilter{GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypePentest}}
+ if passesSearchFilter(answer, f) {
+ t.Fatal("doc without guideType must be blocked when guideType filter is set")
}
})
- t.Run("flowID sets flow_id", func(t *testing.T) {
- f := buildSearchFilters(0, &model.KnowledgeFilter{FlowID: ptr(int64(55))})
- if f["flow_id"] != "55" {
- t.Fatalf("expected flow_id=55, got %v", f["flow_id"])
+ t.Run("matching answerType passes", func(t *testing.T) {
+ f := &model.KnowledgeFilter{AnswerTypes: []model.KnowledgeAnswerType{model.KnowledgeAnswerTypeVulnerability}}
+ if !passesSearchFilter(answer, f) {
+ t.Fatal("matching answerType must pass")
}
})
- t.Run("userID and all single filters combined", func(t *testing.T) {
- f := buildSearchFilters(7, &model.KnowledgeFilter{
- DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeAnswer},
- AnswerTypes: []model.KnowledgeAnswerType{model.KnowledgeAnswerTypeGuide},
- FlowID: ptr(int64(3)),
- })
- if f["user_id"] != "7" || f["doc_type"] != "answer" || f["answer_type"] != "guide" || f["flow_id"] != "3" {
- t.Fatalf("combined filters mismatch: %v", f)
+ t.Run("non-matching answerType blocks", func(t *testing.T) {
+ f := &model.KnowledgeFilter{AnswerTypes: []model.KnowledgeAnswerType{model.KnowledgeAnswerTypeCode}}
+ if passesSearchFilter(answer, f) {
+ t.Fatal("non-matching answerType must block")
+ }
+ })
+
+ t.Run("matching codeLang passes", func(t *testing.T) {
+ f := &model.KnowledgeFilter{CodeLangs: []string{"python"}}
+ if !passesSearchFilter(code, f) {
+ t.Fatal("matching codeLang must pass")
+ }
+ })
+
+ t.Run("non-matching codeLang blocks", func(t *testing.T) {
+ f := &model.KnowledgeFilter{CodeLangs: []string{"go"}}
+ if passesSearchFilter(code, f) {
+ t.Fatal("non-matching codeLang must block")
+ }
+ })
+
+ t.Run("manual=true filter passes manual doc", func(t *testing.T) {
+ f := &model.KnowledgeFilter{Manual: ptr(true)}
+ if !passesSearchFilter(answer, f) {
+ t.Fatal("manual=true filter must pass manual doc")
+ }
+ })
+
+ t.Run("manual=true filter blocks non-manual doc", func(t *testing.T) {
+ f := &model.KnowledgeFilter{Manual: ptr(true)}
+ if passesSearchFilter(guide, f) {
+ t.Fatal("manual=true filter must block non-manual doc")
+ }
+ })
+
+ t.Run("manual=false filter passes non-manual doc", func(t *testing.T) {
+ f := &model.KnowledgeFilter{Manual: ptr(false)}
+ if !passesSearchFilter(guide, f) {
+ t.Fatal("manual=false filter must pass non-manual doc")
+ }
+ })
+
+ t.Run("manual=false filter blocks manual doc", func(t *testing.T) {
+ f := &model.KnowledgeFilter{Manual: ptr(false)}
+ if passesSearchFilter(answer, f) {
+ t.Fatal("manual=false filter must block manual doc")
+ }
+ })
+
+ t.Run("flowID match passes", func(t *testing.T) {
+ f := &model.KnowledgeFilter{FlowID: ptr(int64(10))}
+ if !passesSearchFilter(guide, f) {
+ t.Fatal("matching flowID must pass")
+ }
+ })
+
+ t.Run("flowID mismatch blocks", func(t *testing.T) {
+ f := &model.KnowledgeFilter{FlowID: ptr(int64(99))}
+ if passesSearchFilter(guide, f) {
+ t.Fatal("non-matching flowID must block")
+ }
+ })
+
+ t.Run("flowID filter blocks doc without flowID", func(t *testing.T) {
+ f := &model.KnowledgeFilter{FlowID: ptr(int64(10))}
+ if passesSearchFilter(answer, f) {
+ t.Fatal("doc without flowID must be blocked when flowID filter is set")
+ }
+ })
+
+ t.Run("combined filters all match", func(t *testing.T) {
+ f := &model.KnowledgeFilter{
+ DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide},
+ GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypePentest},
+ FlowID: ptr(int64(10)),
+ }
+ if !passesSearchFilter(guide, f) {
+ t.Fatal("all-matching combined filter must pass")
+ }
+ })
+
+ t.Run("combined filters partial mismatch blocks", func(t *testing.T) {
+ f := &model.KnowledgeFilter{
+ DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide},
+ FlowID: ptr(int64(99)), // wrong flowID
+ }
+ if passesSearchFilter(guide, f) {
+ t.Fatal("partial mismatch in combined filter must block")
+ }
+ })
+}
+
+// ============================================================================
+// Tests: SearchDocuments / SearchUserDocuments
+// ============================================================================
+
+func TestSearchDocuments(t *testing.T) {
+ ctx := context.Background()
+
+ t.Run("embedder unavailable returns error", func(t *testing.T) {
+ ks := &knowledgeStore{embedder: &mockEmbedder{available: false}}
+ _, err := ks.SearchDocuments(ctx, "query", nil, 5)
+ if err == nil {
+ t.Fatal("expected error when embedder unavailable")
+ }
+ })
+
+ t.Run("nil embedder returns error", func(t *testing.T) {
+ ks := &knowledgeStore{embedder: nil}
+ _, err := ks.SearchDocuments(ctx, "query", nil, 5)
+ if err == nil {
+ t.Fatal("expected error when embedder is nil")
+ }
+ })
+
+ t.Run("EmbedDocuments error propagates", func(t *testing.T) {
+ ks := &knowledgeStore{
+ embedder: &mockEmbedder{
+ available: true,
+ embedDocumentsFn: func(_ context.Context, _ []string) ([][]float32, error) {
+ return nil, errors.New("embed error")
+ },
+ },
+ }
+ _, err := ks.SearchDocuments(ctx, "query", nil, 5)
+ if err == nil {
+ t.Fatal("expected error from EmbedDocuments")
+ }
+ })
+
+ t.Run("embedder returning empty vectors is an error", func(t *testing.T) {
+ ks := &knowledgeStore{
+ embedder: &mockEmbedder{
+ available: true,
+ embedDocumentsFn: func(_ context.Context, _ []string) ([][]float32, error) {
+ return [][]float32{}, nil
+ },
+ },
+ }
+ _, err := ks.SearchDocuments(ctx, "query", nil, 5)
+ if err == nil {
+ t.Fatal("expected error for empty vectors")
+ }
+ })
+
+ t.Run("returns results with correct IDs and scores", func(t *testing.T) {
+ db := &mockDB{
+ searchKnowledge: func(_ context.Context, _ database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {
+ return []database.SearchKnowledgeDocumentsRow{
+ makeSearchRow("uuid-1", "content1", `{"doc_type":"answer"}`, 0.95),
+ makeSearchRow("uuid-2", "content2", `{"doc_type":"guide"}`, 0.80),
+ }, nil
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ results, err := ks.SearchDocuments(ctx, "test query", nil, 5)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 2 {
+ t.Fatalf("want 2 results, got %d", len(results))
+ }
+ // Critical: IDs must be populated (this was the original bug)
+ if results[0].Document.ID != "uuid-1" {
+ t.Fatalf("ID mismatch: want uuid-1, got %q", results[0].Document.ID)
+ }
+ if results[1].Document.ID != "uuid-2" {
+ t.Fatalf("ID mismatch: want uuid-2, got %q", results[1].Document.ID)
+ }
+ if results[0].Score != 0.95 {
+ t.Fatalf("score mismatch: want 0.95, got %f", results[0].Score)
+ }
+ if results[1].Score != 0.80 {
+ t.Fatalf("score mismatch: want 0.80, got %f", results[1].Score)
+ }
+ })
+
+ t.Run("correct params passed to db (threshold, limit, vector literal)", func(t *testing.T) {
+ var gotParams database.SearchKnowledgeDocumentsParams
+ db := &mockDB{
+ searchKnowledge: func(_ context.Context, arg database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {
+ gotParams = arg
+ return nil, nil
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ _, err := ks.SearchDocuments(ctx, "q", nil, 7)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if gotParams.MaxDistance != float64(1.0-defaultSearchThreshold) {
+ t.Fatalf("distance threshold: want %f, got %f", float64(1.0-defaultSearchThreshold), gotParams.MaxDistance)
+ }
+ if gotParams.Lim != 7 {
+ t.Fatalf("limit: want 7, got %d", gotParams.Lim)
+ }
+ vec, ok := gotParams.Embedding.(string)
+ if !ok || len(vec) < 2 || vec[0] != '[' {
+ t.Fatalf("vector literal format wrong: %v", gotParams.Embedding)
+ }
+ })
+
+ t.Run("default limit applied when limit<=0", func(t *testing.T) {
+ var gotLimit int32
+ db := &mockDB{
+ searchKnowledge: func(_ context.Context, arg database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {
+ gotLimit = arg.Lim
+ return nil, nil
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ _, err := ks.SearchDocuments(ctx, "q", nil, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if gotLimit != int32(defaultSearchLimit) {
+ t.Fatalf("default limit: want %d, got %d", defaultSearchLimit, gotLimit)
+ }
+ })
+
+ t.Run("go filter by docType applied after db fetch", func(t *testing.T) {
+ db := &mockDB{
+ searchKnowledge: func(_ context.Context, _ database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {
+ return []database.SearchKnowledgeDocumentsRow{
+ makeSearchRow("g1", "", `{"doc_type":"guide"}`, 0.9),
+ makeSearchRow("a1", "", `{"doc_type":"answer"}`, 0.8),
+ }, nil
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ results, err := ks.SearchDocuments(ctx, "q",
+ &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide}}, 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 1 || results[0].Document.ID != "g1" {
+ t.Fatalf("go filter should have kept only guide doc, got %d results", len(results))
+ }
+ })
+
+ t.Run("go filter by flowID applied after db fetch", func(t *testing.T) {
+ db := &mockDB{
+ searchKnowledge: func(_ context.Context, _ database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {
+ return []database.SearchKnowledgeDocumentsRow{
+ makeSearchRow("f10", "", `{"doc_type":"answer","flow_id":10}`, 0.9),
+ makeSearchRow("f20", "", `{"doc_type":"answer","flow_id":20}`, 0.8),
+ }, nil
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ results, err := ks.SearchDocuments(ctx, "q",
+ &model.KnowledgeFilter{FlowID: ptr(int64(10))}, 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 1 || results[0].Document.ID != "f10" {
+ t.Fatalf("flowID filter: expected f10, got %d results", len(results))
+ }
+ })
+
+ t.Run("db error propagates", func(t *testing.T) {
+ db := &mockDB{
+ searchKnowledge: func(_ context.Context, _ database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {
+ return nil, errors.New("db error")
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ _, err := ks.SearchDocuments(ctx, "q", nil, 5)
+ if err == nil {
+ t.Fatal("expected error from db")
+ }
+ })
+}
+
+func TestSearchUserDocuments(t *testing.T) {
+ ctx := context.Background()
+ const userID = int64(42)
+
+ t.Run("calls SearchUserKnowledgeDocuments with correct userID", func(t *testing.T) {
+ var gotParams database.SearchUserKnowledgeDocumentsParams
+ db := &mockDB{
+ searchUserKnowledge: func(_ context.Context, arg database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {
+ gotParams = arg
+ return []database.SearchUserKnowledgeDocumentsRow{
+ makeUserSearchRow("uuid-u1", "content", `{"doc_type":"answer","user_id":42}`, 0.88),
+ }, nil
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+
+ results, err := ks.SearchUserDocuments(ctx, userID, "query", nil, 3)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 1 {
+ t.Fatalf("want 1 result, got %d", len(results))
+ }
+ if results[0].Document.ID != "uuid-u1" {
+ t.Fatalf("ID mismatch: want uuid-u1, got %q", results[0].Document.ID)
+ }
+ if results[0].Score != 0.88 {
+ t.Fatalf("score mismatch: want 0.88, got %f", results[0].Score)
+ }
+ if gotParams.UserID.String != "42" {
+ t.Fatalf("userID param: want 42, got %q", gotParams.UserID.String)
+ }
+ if gotParams.Lim != 3 {
+ t.Fatalf("limit: want 3, got %d", gotParams.Lim)
+ }
+ })
+
+ // SECURITY: the user_id must always be sent to the database layer so the
+ // database can enforce ownership — it must never be omitted or zero.
+ t.Run("SECURITY: userID is always sent to DB", func(t *testing.T) {
+ var capturedUserID string
+ db := &mockDB{
+ searchUserKnowledge: func(_ context.Context, arg database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {
+ capturedUserID = arg.UserID.String
+ return nil, nil
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ _, err := ks.SearchUserDocuments(ctx, userID, "q", nil, 5)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if capturedUserID != "42" {
+ t.Fatalf("SECURITY: user_id must always be passed to DB, got %q", capturedUserID)
+ }
+ })
+
+ t.Run("go filter applied for user search", func(t *testing.T) {
+ db := &mockDB{
+ searchUserKnowledge: func(_ context.Context, _ database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {
+ return []database.SearchUserKnowledgeDocumentsRow{
+ makeUserSearchRow("c1", "", `{"doc_type":"code","code_lang":"go"}`, 0.9),
+ makeUserSearchRow("c2", "", `{"doc_type":"code","code_lang":"python"}`, 0.7),
+ }, nil
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ results, err := ks.SearchUserDocuments(ctx, userID, "q",
+ &model.KnowledgeFilter{CodeLangs: []string{"go"}}, 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(results) != 1 || results[0].Document.ID != "c1" {
+ t.Fatalf("codeLang filter: expected c1 only, got %d results", len(results))
+ }
+ })
+
+ t.Run("db error propagates", func(t *testing.T) {
+ db := &mockDB{
+ searchUserKnowledge: func(_ context.Context, _ database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {
+ return nil, errors.New("db error")
+ },
+ }
+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}
+ _, err := ks.SearchUserDocuments(ctx, userID, "q", nil, 5)
+ if err == nil {
+ t.Fatal("expected error from db")
}
})
}
@@ -856,8 +1253,8 @@ func TestGetUserDocument(t *testing.T) {
if gotParams.Uuid != "doc-uuid" {
t.Fatalf("uuid mismatch: %q", gotParams.Uuid)
}
- if gotParams.Cmetadata.String != "7" {
- t.Fatalf("userID param mismatch: %q", gotParams.Cmetadata.String)
+ if gotParams.UserID.String != "7" {
+ t.Fatalf("userID param mismatch: %q", gotParams.UserID.String)
}
if doc.ID != "doc-uuid" {
t.Fatal("document ID mismatch")
@@ -976,11 +1373,11 @@ func TestDeleteUserDocument(t *testing.T) {
t.Fatal(err)
}
// Ownership enforced: correct uuid and userID sent to DB
- if gotDelParams.Column1.String != "my-doc" {
- t.Fatalf("uuid param: want my-doc, got %q", gotDelParams.Column1.String)
+ if gotDelParams.Uuid.String != "my-doc" {
+ t.Fatalf("uuid param: want my-doc, got %q", gotDelParams.Uuid.String)
}
- if gotDelParams.Column2.String != "3" {
- t.Fatalf("user_id param: want 3, got %q", gotDelParams.Column2.String)
+ if gotDelParams.UserID.String != "3" {
+ t.Fatalf("user_id param: want 3, got %q", gotDelParams.UserID.String)
}
if len(pub.deletedDocs) != 1 {
t.Fatal("event not published")
@@ -1204,8 +1601,8 @@ func TestUpdateDocument(t *testing.T) {
},
updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {
return database.UpdateKnowledgeDocumentRow{
- ID: arg.Column1.String,
- Document: arg.Column3.String,
+ ID: arg.Uuid.String,
+ Document: arg.Document.String,
Cmetadata: sql.NullString{String: existingMeta, Valid: true},
}, nil
},
@@ -1296,8 +1693,8 @@ func TestUpdateDocument(t *testing.T) {
updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {
gotParams = arg
return database.UpdateKnowledgeDocumentRow{
- ID: arg.Column1.String,
- Document: arg.Column3.String,
+ ID: arg.Uuid.String,
+ Document: arg.Document.String,
Cmetadata: sql.NullString{
String: `{"doc_type":"guide","guide_type":"pentest","question":"new q"}`,
Valid: true,
@@ -1320,16 +1717,16 @@ func TestUpdateDocument(t *testing.T) {
}
// Correct uuid passed to update
- if gotParams.Column1.String != "doc-id" {
- t.Fatalf("uuid mismatch: %q", gotParams.Column1.String)
+ if gotParams.Uuid.String != "doc-id" {
+ t.Fatalf("uuid mismatch: %q", gotParams.Uuid.String)
}
// New content passed
- if gotParams.Column3.String != "new content" {
- t.Fatalf("document mismatch: %q", gotParams.Column3.String)
+ if gotParams.Document.String != "new content" {
+ t.Fatalf("document mismatch: %q", gotParams.Document.String)
}
// Embedding formatted as vector literal
- if gotParams.Column2 == "" || gotParams.Column2[0] != '[' {
- t.Fatalf("embedding format wrong: %q", gotParams.Column2)
+ if gotParams.Embedding == "" || gotParams.Embedding[0] != '[' {
+ t.Fatalf("embedding format wrong: %q", gotParams.Embedding)
}
// Updated doc returned
@@ -1359,12 +1756,12 @@ func TestUpdateDocumentDocTypeChange(t *testing.T) {
return makeRow(uuid, "old", initialMeta), nil
},
updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {
- *stored = string(arg.Column4.RawMessage)
+ *stored = string(arg.Cmetadata.RawMessage)
return database.UpdateKnowledgeDocumentRow{
- ID: arg.Column1.String,
- Document: arg.Column3.String,
+ ID: arg.Uuid.String,
+ Document: arg.Document.String,
Cmetadata: sql.NullString{
- String: string(arg.Column4.RawMessage),
+ String: string(arg.Cmetadata.RawMessage),
Valid: true,
},
}, nil
@@ -1578,12 +1975,12 @@ func TestUpdateDocumentSizeCalculation(t *testing.T) {
return row, nil
},
updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {
- *stored = string(arg.Column4.RawMessage)
+ *stored = string(arg.Cmetadata.RawMessage)
return database.UpdateKnowledgeDocumentRow{
- ID: arg.Column1.String,
- Document: arg.Column3.String,
+ ID: arg.Uuid.String,
+ Document: arg.Document.String,
Cmetadata: sql.NullString{
- String: string(arg.Column4.RawMessage),
+ String: string(arg.Cmetadata.RawMessage),
Valid: true,
},
}, nil
@@ -1702,12 +2099,12 @@ func TestUpdateDocumentPreservesOriginalOwner(t *testing.T) {
return makeRow(uuid, "old", originalMeta), nil
},
updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {
- storedMeta = string(arg.Column4.RawMessage)
+ storedMeta = string(arg.Cmetadata.RawMessage)
return database.UpdateKnowledgeDocumentRow{
- ID: arg.Column1.String,
- Document: arg.Column3.String,
+ ID: arg.Uuid.String,
+ Document: arg.Document.String,
Cmetadata: sql.NullString{
- String: string(arg.Column4.RawMessage),
+ String: string(arg.Cmetadata.RawMessage),
Valid: true,
},
}, nil
@@ -1755,7 +2152,7 @@ func TestUpdateUserDocument(t *testing.T) {
db := &mockDB{
getUserKnowledge: func(_ context.Context, arg database.GetUserKnowledgeDocumentParams) (database.GetUserKnowledgeDocumentRow, error) {
fetchedUUID = arg.Uuid
- fetchedUserID = arg.Cmetadata.String
+ fetchedUserID = arg.UserID.String
return database.GetUserKnowledgeDocumentRow{
ID: arg.Uuid,
Document: "old",
@@ -1764,8 +2161,8 @@ func TestUpdateUserDocument(t *testing.T) {
},
updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {
return database.UpdateKnowledgeDocumentRow{
- ID: arg.Column1.String,
- Document: arg.Column3.String,
+ ID: arg.Uuid.String,
+ Document: arg.Document.String,
Cmetadata: sql.NullString{
String: existingMeta,
Valid: true,
diff --git a/backend/pkg/database/querier.go b/backend/pkg/database/querier.go
index 0cfa6e73..6f6335b1 100644
--- a/backend/pkg/database/querier.go
+++ b/backend/pkg/database/querier.go
@@ -40,12 +40,12 @@ type Querier interface {
DeleteFlowAssistantLog(ctx context.Context, id int64) error
// Delete all memory-type documents for a specific flow.
// Called on flow deletion to free long-term memory that will never be re-used.
- // $1 is the decimal text representation of the flow ID (e.g. "55"), matching the
+ // flow_id is the decimal text representation of the flow ID (e.g. "55"), matching the
// text result of (cmetadata ->> 'flow_id') which uses JSON ->> extraction.
- DeleteFlowMemoryDocuments(ctx context.Context, dollar_1 sql.NullString) error
+ DeleteFlowMemoryDocuments(ctx context.Context, flowID sql.NullString) error
DeleteFlowTemplate(ctx context.Context, arg DeleteFlowTemplateParams) error
// Delete a knowledge document by UUID (admin — no user_id check).
- DeleteKnowledgeDocument(ctx context.Context, dollar_1 sql.NullString) error
+ DeleteKnowledgeDocument(ctx context.Context, uuid sql.NullString) error
DeletePrompt(ctx context.Context, id int64) error
DeleteProvider(ctx context.Context, id int64) (Provider, error)
DeleteSubtask(ctx context.Context, id int64) error
@@ -238,9 +238,23 @@ type Querier interface {
// List all knowledge documents excluding the noisy memory type (admin view).
ListAllKnowledgeDocuments(ctx context.Context) ([]ListAllKnowledgeDocumentsRow, error)
// List non-memory knowledge documents belonging to a specific flow (admin scoped).
- ListFlowKnowledgeDocuments(ctx context.Context, cmetadata sql.NullString) ([]ListFlowKnowledgeDocumentsRow, error)
+ ListFlowKnowledgeDocuments(ctx context.Context, flowID sql.NullString) ([]ListFlowKnowledgeDocumentsRow, error)
// List all non-memory knowledge documents owned by a specific user (user-scoped view).
- ListUserKnowledgeDocuments(ctx context.Context, cmetadata sql.NullString) ([]ListUserKnowledgeDocumentsRow, error)
+ ListUserKnowledgeDocuments(ctx context.Context, userID sql.NullString) ([]ListUserKnowledgeDocumentsRow, error)
+ // Vector similarity search over all knowledge documents (admin view, no user filter).
+ // Returns rows ordered by cosine similarity descending (highest score first).
+ // embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'
+ // max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold),
+ // e.g. pass 0.8 to get documents with similarity score > 0.2
+ // lim maximum number of rows to return
+ SearchKnowledgeDocuments(ctx context.Context, arg SearchKnowledgeDocumentsParams) ([]SearchKnowledgeDocumentsRow, error)
+ // Vector similarity search scoped to a specific user (by cmetadata user_id).
+ // Returns rows ordered by cosine similarity descending (highest score first).
+ // embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'
+ // max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold)
+ // lim maximum number of rows to return
+ // user_id owner filter as a decimal text string (e.g. "42")
+ SearchUserKnowledgeDocuments(ctx context.Context, arg SearchUserKnowledgeDocumentsParams) ([]SearchUserKnowledgeDocumentsRow, error)
UpdateAPIToken(ctx context.Context, arg UpdateAPITokenParams) (ApiToken, error)
UpdateAssistant(ctx context.Context, arg UpdateAssistantParams) (Assistant, error)
UpdateAssistantLanguage(ctx context.Context, arg UpdateAssistantLanguageParams) (Assistant, error)
@@ -263,8 +277,8 @@ type Querier interface {
UpdateFlowTitle(ctx context.Context, arg UpdateFlowTitleParams) (Flow, error)
UpdateFlowToolCallIDTemplate(ctx context.Context, arg UpdateFlowToolCallIDTemplateParams) (Flow, error)
// Update an existing document's embedding, text and metadata atomically.
- // $2 must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'
- // $4 must be valid JSON text.
+ // embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'
+ // cmetadata must be valid JSON text.
UpdateKnowledgeDocument(ctx context.Context, arg UpdateKnowledgeDocumentParams) (UpdateKnowledgeDocumentRow, error)
UpdateMsgChain(ctx context.Context, arg UpdateMsgChainParams) (Msgchain, error)
UpdateMsgChainUsage(ctx context.Context, arg UpdateMsgChainUsageParams) (Msgchain, error)
diff --git a/backend/sqlc/models/knowledge.sql b/backend/sqlc/models/knowledge.sql
index 1c859dc0..a0812409 100644
--- a/backend/sqlc/models/knowledge.sql
+++ b/backend/sqlc/models/knowledge.sql
@@ -7,7 +7,7 @@ SELECT
FROM langchain_pg_embedding e
INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid
WHERE c.name = 'langchain'
- AND e.uuid::text = $1;
+ AND e.uuid::text = sqlc.arg(uuid);
-- name: GetUserKnowledgeDocument :one
-- Fetch a single knowledge document by UUID, scoped to a specific user.
@@ -18,8 +18,8 @@ SELECT
FROM langchain_pg_embedding e
INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid
WHERE c.name = 'langchain'
- AND e.uuid::text = $1
- AND (e.cmetadata ->> 'user_id') = $2;
+ AND e.uuid::text = sqlc.arg(uuid)
+ AND (e.cmetadata ->> 'user_id') = sqlc.arg(user_id);
-- name: ListAllKnowledgeDocuments :many
-- List all knowledge documents excluding the noisy memory type (admin view).
@@ -43,7 +43,7 @@ FROM langchain_pg_embedding e
INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid
WHERE c.name = 'langchain'
AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')
- AND (e.cmetadata ->> 'flow_id') = $1
+ AND (e.cmetadata ->> 'flow_id') = sqlc.arg(flow_id)
ORDER BY e.uuid;
-- name: ListUserKnowledgeDocuments :many
@@ -56,19 +56,19 @@ FROM langchain_pg_embedding e
INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid
WHERE c.name = 'langchain'
AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')
- AND (e.cmetadata ->> 'user_id') = $1
+ AND (e.cmetadata ->> 'user_id') = sqlc.arg(user_id)
ORDER BY e.uuid;
-- name: UpdateKnowledgeDocument :one
-- Update an existing document's embedding, text and metadata atomically.
--- $2 must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'
--- $4 must be valid JSON text.
+-- embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'
+-- cmetadata must be valid JSON text.
UPDATE langchain_pg_embedding
SET
- embedding = $2::vector,
- document = $3,
- cmetadata = $4::json
-WHERE uuid::text = $1
+ embedding = sqlc.arg(embedding)::vector,
+ document = sqlc.arg(document),
+ cmetadata = sqlc.arg(cmetadata)::json
+WHERE uuid::text = sqlc.arg(uuid)
AND collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain')
RETURNING
uuid::text AS id,
@@ -78,22 +78,63 @@ RETURNING
-- name: DeleteKnowledgeDocument :exec
-- Delete a knowledge document by UUID (admin — no user_id check).
DELETE FROM langchain_pg_embedding
-WHERE uuid::text = $1
+WHERE uuid::text = sqlc.arg(uuid)
AND collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain');
-- name: DeleteUserKnowledgeDocument :exec
-- Delete a knowledge document by UUID, only if it belongs to the given user.
DELETE FROM langchain_pg_embedding
-WHERE uuid::text = $1
- AND (cmetadata ->> 'user_id') = $2
+WHERE uuid::text = sqlc.arg(uuid)
+ AND (cmetadata ->> 'user_id') = sqlc.arg(user_id)
AND collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain');
+-- name: SearchKnowledgeDocuments :many
+-- Vector similarity search over all knowledge documents (admin view, no user filter).
+-- Returns rows ordered by cosine similarity descending (highest score first).
+-- embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'
+-- max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold),
+-- e.g. pass 0.8 to get documents with similarity score > 0.2
+-- lim maximum number of rows to return
+SELECT
+ e.uuid::text AS id,
+ COALESCE(e.document, '') AS document,
+ COALESCE(e.cmetadata::text, '{}') AS cmetadata,
+ (1.0 - (e.embedding <=> sqlc.arg(embedding)::vector))::float8 AS score
+FROM langchain_pg_embedding e
+INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid
+WHERE c.name = 'langchain'
+ AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')
+ AND (e.embedding <=> sqlc.arg(embedding)::vector)::float8 < sqlc.arg(max_distance)::float8
+ORDER BY e.embedding <=> sqlc.arg(embedding)::vector
+LIMIT sqlc.arg(lim)::int;
+
+-- name: SearchUserKnowledgeDocuments :many
+-- Vector similarity search scoped to a specific user (by cmetadata user_id).
+-- Returns rows ordered by cosine similarity descending (highest score first).
+-- embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'
+-- max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold)
+-- lim maximum number of rows to return
+-- user_id owner filter as a decimal text string (e.g. "42")
+SELECT
+ e.uuid::text AS id,
+ COALESCE(e.document, '') AS document,
+ COALESCE(e.cmetadata::text, '{}') AS cmetadata,
+ (1.0 - (e.embedding <=> sqlc.arg(embedding)::vector))::float8 AS score
+FROM langchain_pg_embedding e
+INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid
+WHERE c.name = 'langchain'
+ AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')
+ AND (e.cmetadata ->> 'user_id') = sqlc.arg(user_id)
+ AND (e.embedding <=> sqlc.arg(embedding)::vector)::float8 < sqlc.arg(max_distance)::float8
+ORDER BY e.embedding <=> sqlc.arg(embedding)::vector
+LIMIT sqlc.arg(lim)::int;
+
-- name: DeleteFlowMemoryDocuments :exec
-- Delete all memory-type documents for a specific flow.
-- Called on flow deletion to free long-term memory that will never be re-used.
--- $1 is the decimal text representation of the flow ID (e.g. "55"), matching the
+-- flow_id is the decimal text representation of the flow ID (e.g. "55"), matching the
-- text result of (cmetadata ->> 'flow_id') which uses JSON ->> extraction.
DELETE FROM langchain_pg_embedding
WHERE collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain')
AND COALESCE(cmetadata ->> 'doc_type', '') = 'memory'
- AND (cmetadata ->> 'flow_id') = $1;
+ AND (cmetadata ->> 'flow_id') = sqlc.arg(flow_id);