Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
fix(database/knowledge): replace SimilaritySearch with direct SQLC vector queries
Dmitry Ng0f0a7bd2d0ab
diff --git a/backend/pkg/database/knowledge.sql.go b/backend/pkg/database/knowledge.sql.goindex ddbfb7bf..b6c6c8db 100644--- a/backend/pkg/database/knowledge.sql.go+++ b/backend/pkg/database/knowledge.sql.go@@ -21,10 +21,10 @@ WHERE collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'la// Delete all memory-type documents for a specific flow.// Called on flow deletion to free long-term memory that will never be re-used.-// $1 is the decimal text representation of the flow ID (e.g. "55"), matching the+// flow_id is the decimal text representation of the flow ID (e.g. "55"), matching the// text result of (cmetadata ->> 'flow_id') which uses JSON ->> extraction.-func (q *Queries) DeleteFlowMemoryDocuments(ctx context.Context, dollar_1 sql.NullString) error {- _, err := q.db.ExecContext(ctx, deleteFlowMemoryDocuments, dollar_1)+func (q *Queries) DeleteFlowMemoryDocuments(ctx context.Context, flowID sql.NullString) error {+ _, err := q.db.ExecContext(ctx, deleteFlowMemoryDocuments, flowID)return err}@@ -35,8 +35,8 @@ WHERE uuid::text = $1`// Delete a knowledge document by UUID (admin — no user_id check).-func (q *Queries) DeleteKnowledgeDocument(ctx context.Context, dollar_1 sql.NullString) error {- _, err := q.db.ExecContext(ctx, deleteKnowledgeDocument, dollar_1)+func (q *Queries) DeleteKnowledgeDocument(ctx context.Context, uuid sql.NullString) error {+ _, err := q.db.ExecContext(ctx, deleteKnowledgeDocument, uuid)return err}@@ -48,13 +48,13 @@ WHERE uuid::text = $1`type DeleteUserKnowledgeDocumentParams struct {- Column1 sql.NullString `json:"column_1"`- Column2 sql.NullString `json:"column_2"`+ Uuid sql.NullString `json:"uuid"`+ UserID sql.NullString `json:"user_id"`}// Delete a knowledge document by UUID, only if it belongs to the given user.func (q *Queries) DeleteUserKnowledgeDocument(ctx context.Context, arg DeleteUserKnowledgeDocumentParams) error {- _, err := q.db.ExecContext(ctx, deleteUserKnowledgeDocument, arg.Column1, arg.Column2)+ _, err := q.db.ExecContext(ctx, deleteUserKnowledgeDocument, arg.Uuid, arg.UserID)return err}@@ -96,8 +96,8 @@ WHERE c.name = 'langchain'`type GetUserKnowledgeDocumentParams struct {- Uuid string `json:"uuid"`- Cmetadata sql.NullString `json:"cmetadata"`+ Uuid string `json:"uuid"`+ UserID sql.NullString `json:"user_id"`}type GetUserKnowledgeDocumentRow struct {@@ -108,7 +108,7 @@ type GetUserKnowledgeDocumentRow struct {// Fetch a single knowledge document by UUID, scoped to a specific user.func (q *Queries) GetUserKnowledgeDocument(ctx context.Context, arg GetUserKnowledgeDocumentParams) (GetUserKnowledgeDocumentRow, error) {- row := q.db.QueryRowContext(ctx, getUserKnowledgeDocument, arg.Uuid, arg.Cmetadata)+ row := q.db.QueryRowContext(ctx, getUserKnowledgeDocument, arg.Uuid, arg.UserID)var i GetUserKnowledgeDocumentRowerr := row.Scan(&i.ID, &i.Document, &i.Cmetadata)return i, err@@ -176,8 +176,8 @@ type ListFlowKnowledgeDocumentsRow struct {}// List non-memory knowledge documents belonging to a specific flow (admin scoped).-func (q *Queries) ListFlowKnowledgeDocuments(ctx context.Context, cmetadata sql.NullString) ([]ListFlowKnowledgeDocumentsRow, error) {- rows, err := q.db.QueryContext(ctx, listFlowKnowledgeDocuments, cmetadata)+func (q *Queries) ListFlowKnowledgeDocuments(ctx context.Context, flowID sql.NullString) ([]ListFlowKnowledgeDocumentsRow, error) {+ rows, err := q.db.QueryContext(ctx, listFlowKnowledgeDocuments, flowID)if err != nil {return nil, err}@@ -219,8 +219,8 @@ type ListUserKnowledgeDocumentsRow struct {}// List all non-memory knowledge documents owned by a specific user (user-scoped view).-func (q *Queries) ListUserKnowledgeDocuments(ctx context.Context, cmetadata sql.NullString) ([]ListUserKnowledgeDocumentsRow, error) {- rows, err := q.db.QueryContext(ctx, listUserKnowledgeDocuments, cmetadata)+func (q *Queries) ListUserKnowledgeDocuments(ctx context.Context, userID sql.NullString) ([]ListUserKnowledgeDocumentsRow, error) {+ rows, err := q.db.QueryContext(ctx, listUserKnowledgeDocuments, userID)if err != nil {return nil, err}@@ -242,13 +242,146 @@ func (q *Queries) ListUserKnowledgeDocuments(ctx context.Context, cmetadata sql.return items, nil}+const searchKnowledgeDocuments = `-- name: SearchKnowledgeDocuments :many+SELECT+ e.uuid::text AS id,+ COALESCE(e.document, '') AS document,+ COALESCE(e.cmetadata::text, '{}') AS cmetadata,+ (1.0 - (e.embedding <=> $1::vector))::float8 AS score+FROM langchain_pg_embedding e+INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid+WHERE c.name = 'langchain'+ AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')+ AND (e.embedding <=> $1::vector)::float8 < $2::float8+ORDER BY e.embedding <=> $1::vector+LIMIT $3::int+`++type SearchKnowledgeDocumentsParams struct {+ Embedding interface{} `json:"embedding"`+ MaxDistance float64 `json:"max_distance"`+ Lim int32 `json:"lim"`+}++type SearchKnowledgeDocumentsRow struct {+ ID string `json:"id"`+ Document string `json:"document"`+ Cmetadata sql.NullString `json:"cmetadata"`+ Score float64 `json:"score"`+}++// Vector similarity search over all knowledge documents (admin view, no user filter).+// Returns rows ordered by cosine similarity descending (highest score first).+// embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'+// max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold),+//+// e.g. pass 0.8 to get documents with similarity score > 0.2+//+// lim maximum number of rows to return+func (q *Queries) SearchKnowledgeDocuments(ctx context.Context, arg SearchKnowledgeDocumentsParams) ([]SearchKnowledgeDocumentsRow, error) {+ rows, err := q.db.QueryContext(ctx, searchKnowledgeDocuments, arg.Embedding, arg.MaxDistance, arg.Lim)+ if err != nil {+ return nil, err+ }+ defer rows.Close()+ var items []SearchKnowledgeDocumentsRow+ for rows.Next() {+ var i SearchKnowledgeDocumentsRow+ if err := rows.Scan(+ &i.ID,+ &i.Document,+ &i.Cmetadata,+ &i.Score,+ ); err != nil {+ return nil, err+ }+ items = append(items, i)+ }+ if err := rows.Close(); err != nil {+ return nil, err+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return items, nil+}++const searchUserKnowledgeDocuments = `-- name: SearchUserKnowledgeDocuments :many+SELECT+ e.uuid::text AS id,+ COALESCE(e.document, '') AS document,+ COALESCE(e.cmetadata::text, '{}') AS cmetadata,+ (1.0 - (e.embedding <=> $1::vector))::float8 AS score+FROM langchain_pg_embedding e+INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid+WHERE c.name = 'langchain'+ AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')+ AND (e.cmetadata ->> 'user_id') = $2+ AND (e.embedding <=> $1::vector)::float8 < $3::float8+ORDER BY e.embedding <=> $1::vector+LIMIT $4::int+`++type SearchUserKnowledgeDocumentsParams struct {+ Embedding interface{} `json:"embedding"`+ UserID sql.NullString `json:"user_id"`+ MaxDistance float64 `json:"max_distance"`+ Lim int32 `json:"lim"`+}++type SearchUserKnowledgeDocumentsRow struct {+ ID string `json:"id"`+ Document string `json:"document"`+ Cmetadata sql.NullString `json:"cmetadata"`+ Score float64 `json:"score"`+}++// Vector similarity search scoped to a specific user (by cmetadata user_id).+// Returns rows ordered by cosine similarity descending (highest score first).+// embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'+// max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold)+// lim maximum number of rows to return+// user_id owner filter as a decimal text string (e.g. "42")+func (q *Queries) SearchUserKnowledgeDocuments(ctx context.Context, arg SearchUserKnowledgeDocumentsParams) ([]SearchUserKnowledgeDocumentsRow, error) {+ rows, err := q.db.QueryContext(ctx, searchUserKnowledgeDocuments,+ arg.Embedding,+ arg.UserID,+ arg.MaxDistance,+ arg.Lim,+ )+ if err != nil {+ return nil, err+ }+ defer rows.Close()+ var items []SearchUserKnowledgeDocumentsRow+ for rows.Next() {+ var i SearchUserKnowledgeDocumentsRow+ if err := rows.Scan(+ &i.ID,+ &i.Document,+ &i.Cmetadata,+ &i.Score,+ ); err != nil {+ return nil, err+ }+ items = append(items, i)+ }+ if err := rows.Close(); err != nil {+ return nil, err+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return items, nil+}+const updateKnowledgeDocument = `-- name: UpdateKnowledgeDocument :oneUPDATE langchain_pg_embeddingSET- embedding = $2::vector,- document = $3,- cmetadata = $4::json-WHERE uuid::text = $1+ embedding = $1::vector,+ document = $2,+ cmetadata = $3::json+WHERE uuid::text = $4AND collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain')RETURNINGuuid::text AS id,@@ -257,10 +390,10 @@ RETURNING`type UpdateKnowledgeDocumentParams struct {- Column1 sql.NullString `json:"column_1"`- Column2 string `json:"column_2"`- Column3 sql.NullString `json:"column_3"`- Column4 pqtype.NullRawMessage `json:"column_4"`+ Embedding string `json:"embedding"`+ Document sql.NullString `json:"document"`+ Cmetadata pqtype.NullRawMessage `json:"cmetadata"`+ Uuid sql.NullString `json:"uuid"`}type UpdateKnowledgeDocumentRow struct {@@ -270,14 +403,14 @@ type UpdateKnowledgeDocumentRow struct {}// Update an existing document's embedding, text and metadata atomically.-// $2 must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'-// $4 must be valid JSON text.+// embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'+// cmetadata must be valid JSON text.func (q *Queries) UpdateKnowledgeDocument(ctx context.Context, arg UpdateKnowledgeDocumentParams) (UpdateKnowledgeDocumentRow, error) {row := q.db.QueryRowContext(ctx, updateKnowledgeDocument,- arg.Column1,- arg.Column2,- arg.Column3,- arg.Column4,+ arg.Embedding,+ arg.Document,+ arg.Cmetadata,+ arg.Uuid,)var i UpdateKnowledgeDocumentRowerr := row.Scan(&i.ID, &i.Document, &i.Cmetadata)diff --git a/backend/pkg/database/knowledge/knowledge.go b/backend/pkg/database/knowledge/knowledge.goindex f6dcd8b7..794210a8 100644--- a/backend/pkg/database/knowledge/knowledge.go+++ b/backend/pkg/database/knowledge/knowledge.go@@ -20,7 +20,6 @@ import ()const (- collectionName = "langchain"defaultSearchLimit = 10defaultSearchThreshold = float32(0.2)manualDocumentFlag = "manual"@@ -224,34 +223,6 @@ func applyGoFilters(docs []*model.KnowledgeDocument, filter *model.KnowledgeFiltreturn result}-// buildSearchFilters converts a KnowledgeFilter to a pgvector filter map.-// Only single-value equality filters are expressed here; arrays are applied in Go.-func buildSearchFilters(userID int64, filter *model.KnowledgeFilter) map[string]any {- f := make(map[string]any)- if userID > 0 {- f["user_id"] = strconv.FormatInt(userID, 10)- }- if filter == nil {- return f- }- if len(filter.DocTypes) == 1 {- f["doc_type"] = string(filter.DocTypes[0])- }- if len(filter.GuideTypes) == 1 {- f["guide_type"] = string(filter.GuideTypes[0])- }- if len(filter.AnswerTypes) == 1 {- f["answer_type"] = string(filter.AnswerTypes[0])- }- if len(filter.CodeLangs) == 1 {- f["code_lang"] = filter.CodeLangs[0]- }- if filter.FlowID != nil {- f["flow_id"] = strconv.FormatInt(*filter.FlowID, 10)- }- return f-}-func (ks *knowledgeStore) requireStore() error {if ks.store == nil {return fmt.Errorf("knowledge: embedding provider is not configured")@@ -338,8 +309,8 @@ func (ks *knowledgeStore) GetDocument(ctx context.Context, id string) (*model.Knfunc (ks *knowledgeStore) GetUserDocument(ctx context.Context, userID int64, id string) (*model.KnowledgeDocument, error) {row, err := ks.db.GetUserKnowledgeDocument(ctx, database.GetUserKnowledgeDocumentParams{- Uuid: id,- Cmetadata: nsOf(strconv.FormatInt(userID, 10)),+ Uuid: id,+ UserID: nsOf(strconv.FormatInt(userID, 10)),})if err != nil {return nil, fmt.Errorf("knowledge: get user document %s: %w", id, err)@@ -359,61 +330,108 @@ func (ks *knowledgeStore) SearchUserDocuments(ctx context.Context, userID int64,return ks.doSearch(ctx, userID, query, filter, limit)}+// doSearch performs a parameterised vector similarity search and returns+// matched documents with their cosine-similarity scores and correct UUIDs.+//+// Filters are applied in two layers:+// 1. SQL layer — user_id ownership (when userID > 0); cosine-distance+// threshold; row limit. All values are bound as query parameters so+// there is no risk of SQL injection.+// 2. Go layer — multi-value enum filters (DocTypes, GuideTypes, etc.) and+// the Manual flag that cannot be expressed as static SQL predicates.func (ks *knowledgeStore) doSearch(ctx context.Context, userID int64, query string, filter *model.KnowledgeFilter, limit int) ([]*model.KnowledgeDocumentWithScore, error) {- if err := ks.requireStore(); err != nil {+ if err := ks.requireEmbedder(); err != nil {return nil, err}if limit <= 0 {limit = defaultSearchLimit}- pgFilters := buildSearchFilters(userID, filter)- opts := []vectorstores.Option{- vectorstores.WithScoreThreshold(defaultSearchThreshold),+ // Compute query embedding.+ vecs, err := ks.embedder.EmbedDocuments(ctx, []string{query})+ if err != nil {+ return nil, fmt.Errorf("knowledge: embed query: %w", err)}- if len(pgFilters) > 0 {- opts = append(opts, vectorstores.WithFilters(pgFilters))+ if len(vecs) == 0 {+ return nil, fmt.Errorf("knowledge: embedder returned no vectors for query")}- schemaDocs, err := ks.store.SimilaritySearch(ctx, query, limit, opts...)- if err != nil {- return nil, fmt.Errorf("knowledge: similarity search: %w", err)+ vecLiteral := formatVector(vecs[0])+ // maxDist is the cosine-distance upper bound (exclusive):+ // distance = 1 - similarity, so maxDist = 1 - threshold.+ maxDist := float64(1.0 - defaultSearchThreshold)++ type searchRow struct {+ ID string+ Document string+ Cmetadata sql.NullString+ Score float64}- results := make([]*model.KnowledgeDocumentWithScore, 0, len(schemaDocs))- for _, sd := range schemaDocs {- doc := schemaDocToModel(sd)- if !passesGoFilter(doc, filter) {+ var rawRows []searchRow++ if userID > 0 {+ rows, err := ks.db.SearchUserKnowledgeDocuments(ctx, database.SearchUserKnowledgeDocumentsParams{+ Embedding: vecLiteral,+ UserID: nsOf(strconv.FormatInt(userID, 10)),+ MaxDistance: maxDist,+ Lim: int32(limit), //nolint:gosec+ })+ if err != nil {+ return nil, fmt.Errorf("knowledge: similarity search (user %d): %w", userID, err)+ }+ rawRows = make([]searchRow, len(rows))+ for i, r := range rows {+ rawRows[i] = searchRow{r.ID, r.Document, r.Cmetadata, r.Score}+ }+ } else {+ rows, err := ks.db.SearchKnowledgeDocuments(ctx, database.SearchKnowledgeDocumentsParams{+ Embedding: vecLiteral,+ MaxDistance: maxDist,+ Lim: int32(limit), //nolint:gosec+ })+ if err != nil {+ return nil, fmt.Errorf("knowledge: similarity search (admin): %w", err)+ }+ rawRows = make([]searchRow, len(rows))+ for i, r := range rows {+ rawRows[i] = searchRow{r.ID, r.Document, r.Cmetadata, r.Score}+ }+ }++ results := make([]*model.KnowledgeDocumentWithScore, 0, len(rawRows))+ for _, r := range rawRows {+ doc := rowToModel(r.ID, r.Document, nullStr(r.Cmetadata), true)+ if !passesSearchFilter(doc, filter) {continue}results = append(results, &model.KnowledgeDocumentWithScore{- Score: float64(sd.Score),+ Score: r.Score,Document: doc,})}return results, nil}-func schemaDocToModel(sd schema.Document) *model.KnowledgeDocument {- raw, _ := json.Marshal(sd.Metadata)- meta := parseMeta(string(raw))- return metaToModelDoc("", sd.PageContent, meta)-}--func passesGoFilter(doc *model.KnowledgeDocument, filter *model.KnowledgeFilter) bool {+// passesSearchFilter checks the Go-side filters that cannot be expressed as+// static SQL predicates (FlowID equality, multi-value enums, and the Manual flag).+func passesSearchFilter(doc *model.KnowledgeDocument, filter *model.KnowledgeFilter) bool {if filter == nil {return true}- if len(filter.DocTypes) > 1 && !slices.Contains(filter.DocTypes, doc.DocType) {+ if filter.FlowID != nil && (doc.FlowID == nil || *doc.FlowID != *filter.FlowID) {+ return false+ }+ if len(filter.DocTypes) > 0 && !slices.Contains(filter.DocTypes, doc.DocType) {return false}- if len(filter.GuideTypes) > 1 && (doc.GuideType == nil || !slices.Contains(filter.GuideTypes, *doc.GuideType)) {+ if len(filter.GuideTypes) > 0 && (doc.GuideType == nil || !slices.Contains(filter.GuideTypes, *doc.GuideType)) {return false}- if len(filter.AnswerTypes) > 1 && (doc.AnswerType == nil || !slices.Contains(filter.AnswerTypes, *doc.AnswerType)) {+ if len(filter.AnswerTypes) > 0 && (doc.AnswerType == nil || !slices.Contains(filter.AnswerTypes, *doc.AnswerType)) {return false}- if len(filter.CodeLangs) > 1 && (doc.CodeLang == nil || !slices.Contains(filter.CodeLangs, *doc.CodeLang)) {+ if len(filter.CodeLangs) > 0 && (doc.CodeLang == nil || !slices.Contains(filter.CodeLangs, *doc.CodeLang)) {return false}if filter.Manual != nil && doc.Manual != *filter.Manual {@@ -588,10 +606,10 @@ func (ks *knowledgeStore) doUpdate(ctx context.Context, userID int64, id string,}row, err := ks.db.UpdateKnowledgeDocument(ctx, database.UpdateKnowledgeDocumentParams{- Column1: nsOf(id), // uuid::text = $1- Column2: formatVector(vecs[0]), // embedding = $2::vector- Column3: nsOf(content), // document = $3- Column4: cmJSON, // cmetadata = $4::json+ Uuid: nsOf(id),+ Embedding: formatVector(vecs[0]),+ Document: nsOf(content),+ Cmetadata: cmJSON,})if err != nil {return nil, fmt.Errorf("knowledge: update document %s: %w", id, err)@@ -624,8 +642,8 @@ func (ks *knowledgeStore) DeleteUserDocument(ctx context.Context, userID int64,return err}if err := ks.db.DeleteUserKnowledgeDocument(ctx, database.DeleteUserKnowledgeDocumentParams{- Column1: nsOf(id),- Column2: nsOf(strconv.FormatInt(userID, 10)),+ Uuid: nsOf(id),+ UserID: nsOf(strconv.FormatInt(userID, 10)),}); err != nil {return fmt.Errorf("knowledge: delete user document %s: %w", id, err)}diff --git a/backend/pkg/database/knowledge/knowledge_test.go b/backend/pkg/database/knowledge/knowledge_test.goindex 1ee42fa1..2e2e9c0f 100644--- a/backend/pkg/database/knowledge/knowledge_test.go+++ b/backend/pkg/database/knowledge/knowledge_test.go@@ -33,8 +33,10 @@ type mockDB struct {listFlow func(ctx context.Context, flowID sql.NullString) ([]database.ListFlowKnowledgeDocumentsRow, error)listUser func(ctx context.Context, userID sql.NullString) ([]database.ListUserKnowledgeDocumentsRow, error)updateKnowledge func(ctx context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error)- deleteKnowledge func(ctx context.Context, dollar_1 sql.NullString) error+ deleteKnowledge func(ctx context.Context, uuid sql.NullString) errordeleteUserKnowledge func(ctx context.Context, arg database.DeleteUserKnowledgeDocumentParams) error+ searchKnowledge func(ctx context.Context, arg database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error)+ searchUserKnowledge func(ctx context.Context, arg database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error)}func (m *mockDB) GetKnowledgeDocument(ctx context.Context, uuid string) (database.GetKnowledgeDocumentRow, error) {@@ -55,12 +57,24 @@ func (m *mockDB) ListUserKnowledgeDocuments(ctx context.Context, cmetadata sql.Nfunc (m *mockDB) UpdateKnowledgeDocument(ctx context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {return m.updateKnowledge(ctx, arg)}-func (m *mockDB) DeleteKnowledgeDocument(ctx context.Context, dollar_1 sql.NullString) error {- return m.deleteKnowledge(ctx, dollar_1)+func (m *mockDB) DeleteKnowledgeDocument(ctx context.Context, uuid sql.NullString) error {+ return m.deleteKnowledge(ctx, uuid)}func (m *mockDB) DeleteUserKnowledgeDocument(ctx context.Context, arg database.DeleteUserKnowledgeDocumentParams) error {return m.deleteUserKnowledge(ctx, arg)}+func (m *mockDB) SearchKnowledgeDocuments(ctx context.Context, arg database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {+ if m.searchKnowledge != nil {+ return m.searchKnowledge(ctx, arg)+ }+ return nil, nil+}+func (m *mockDB) SearchUserKnowledgeDocuments(ctx context.Context, arg database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {+ if m.searchUserKnowledge != nil {+ return m.searchUserKnowledge(ctx, arg)+ }+ return nil, nil+}// --- mockVectorStore --------------------------------------------------------@@ -158,6 +172,24 @@ func makeListUserRow(id, document, cmetadata string) database.ListUserKnowledgeD}}+func makeSearchRow(id, document, cmetadata string, score float64) database.SearchKnowledgeDocumentsRow {+ return database.SearchKnowledgeDocumentsRow{+ ID: id,+ Document: document,+ Cmetadata: sql.NullString{String: cmetadata, Valid: true},+ Score: score,+ }+}++func makeUserSearchRow(id, document, cmetadata string, score float64) database.SearchUserKnowledgeDocumentsRow {+ return database.SearchUserKnowledgeDocumentsRow{+ ID: id,+ Document: document,+ Cmetadata: sql.NullString{String: cmetadata, Valid: true},+ Score: score,+ }+}+// newPublisherFactory returns a PublisherFactory backed by a single shared publisher.func newPublisherFactory(pub *mockPublisher) PublisherFactory {return func(userID int64) subscriptions.KnowledgePublisher {@@ -531,83 +563,448 @@ func TestApplyGoFilters(t *testing.T) {}// ============================================================================-// Tests: buildSearchFilters+// Tests: passesSearchFilter// ============================================================================-func TestBuildSearchFilters(t *testing.T) {- t.Run("userID=0 produces no user_id key", func(t *testing.T) {- f := buildSearchFilters(0, nil)- if _, ok := f["user_id"]; ok {- t.Fatal("user_id should be absent when userID=0")+func TestPassesSearchFilter(t *testing.T) {+ flowID10 := int64(10)+ guide := &model.KnowledgeDocument{+ DocType: model.KnowledgeDocTypeGuide,+ GuideType: ptr(model.KnowledgeGuideTypePentest),+ FlowID: &flowID10,+ }+ answer := &model.KnowledgeDocument{+ DocType: model.KnowledgeDocTypeAnswer,+ AnswerType: ptr(model.KnowledgeAnswerTypeVulnerability),+ Manual: true,+ }+ code := &model.KnowledgeDocument{+ DocType: model.KnowledgeDocTypeCode,+ CodeLang: ptr("python"),+ }++ t.Run("nil filter passes everything", func(t *testing.T) {+ if !passesSearchFilter(guide, nil) {+ t.Fatal("nil filter must pass all documents")}})- t.Run("userID>0 adds user_id string", func(t *testing.T) {- f := buildSearchFilters(42, nil)- if f["user_id"] != "42" {- t.Fatalf("expected user_id=42, got %v", f["user_id"])+ t.Run("empty filter passes everything", func(t *testing.T) {+ if !passesSearchFilter(answer, &model.KnowledgeFilter{}) {+ t.Fatal("empty filter must pass all documents")}})- t.Run("nil filter returns only userID-based key", func(t *testing.T) {- f := buildSearchFilters(1, nil)- if len(f) != 1 {- t.Fatalf("want 1 key, got %d", len(f))+ t.Run("matching docType passes", func(t *testing.T) {+ f := &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide}}+ if !passesSearchFilter(guide, f) {+ t.Fatal("matching docType must pass")}})- t.Run("single docType sets doc_type", func(t *testing.T) {- f := buildSearchFilters(0, &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide}})- if f["doc_type"] != "guide" {- t.Fatalf("expected doc_type=guide, got %v", f["doc_type"])+ t.Run("non-matching docType blocks", func(t *testing.T) {+ f := &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeCode}}+ if passesSearchFilter(guide, f) {+ t.Fatal("non-matching docType must block")}})- t.Run("multiple docTypes does NOT set doc_type (handled in Go)", func(t *testing.T) {- f := buildSearchFilters(0, &model.KnowledgeFilter{- DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide, model.KnowledgeDocTypeCode},- })- if _, ok := f["doc_type"]; ok {- t.Fatal("doc_type should not be set for multiple types")+ t.Run("multiple docTypes: match passes", func(t *testing.T) {+ f := &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide, model.KnowledgeDocTypeCode}}+ if !passesSearchFilter(guide, f) {+ t.Fatal("doc matching one of multiple types must pass")}})- t.Run("single guideType sets guide_type", func(t *testing.T) {- f := buildSearchFilters(0, &model.KnowledgeFilter{GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypePentest}})- if f["guide_type"] != "pentest" {- t.Fatalf("expected guide_type=pentest, got %v", f["guide_type"])+ t.Run("matching guideType passes", func(t *testing.T) {+ f := &model.KnowledgeFilter{GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypePentest}}+ if !passesSearchFilter(guide, f) {+ t.Fatal("matching guideType must pass")}})- t.Run("single answerType sets answer_type", func(t *testing.T) {- f := buildSearchFilters(0, &model.KnowledgeFilter{AnswerTypes: []model.KnowledgeAnswerType{model.KnowledgeAnswerTypeCode}})- if f["answer_type"] != "code" {- t.Fatalf("expected answer_type=code, got %v", f["answer_type"])+ t.Run("non-matching guideType blocks", func(t *testing.T) {+ f := &model.KnowledgeFilter{GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypeInstall}}+ if passesSearchFilter(guide, f) {+ t.Fatal("non-matching guideType must block")}})- t.Run("single codeLang sets code_lang", func(t *testing.T) {- f := buildSearchFilters(0, &model.KnowledgeFilter{CodeLangs: []string{"go"}})- if f["code_lang"] != "go" {- t.Fatalf("expected code_lang=go, got %v", f["code_lang"])+ t.Run("doc without guideType is blocked by guideType filter", func(t *testing.T) {+ f := &model.KnowledgeFilter{GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypePentest}}+ if passesSearchFilter(answer, f) {+ t.Fatal("doc without guideType must be blocked when guideType filter is set")}})- t.Run("flowID sets flow_id", func(t *testing.T) {- f := buildSearchFilters(0, &model.KnowledgeFilter{FlowID: ptr(int64(55))})- if f["flow_id"] != "55" {- t.Fatalf("expected flow_id=55, got %v", f["flow_id"])+ t.Run("matching answerType passes", func(t *testing.T) {+ f := &model.KnowledgeFilter{AnswerTypes: []model.KnowledgeAnswerType{model.KnowledgeAnswerTypeVulnerability}}+ if !passesSearchFilter(answer, f) {+ t.Fatal("matching answerType must pass")}})- t.Run("userID and all single filters combined", func(t *testing.T) {- f := buildSearchFilters(7, &model.KnowledgeFilter{- DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeAnswer},- AnswerTypes: []model.KnowledgeAnswerType{model.KnowledgeAnswerTypeGuide},- FlowID: ptr(int64(3)),- })- if f["user_id"] != "7" || f["doc_type"] != "answer" || f["answer_type"] != "guide" || f["flow_id"] != "3" {- t.Fatalf("combined filters mismatch: %v", f)+ t.Run("non-matching answerType blocks", func(t *testing.T) {+ f := &model.KnowledgeFilter{AnswerTypes: []model.KnowledgeAnswerType{model.KnowledgeAnswerTypeCode}}+ if passesSearchFilter(answer, f) {+ t.Fatal("non-matching answerType must block")+ }+ })++ t.Run("matching codeLang passes", func(t *testing.T) {+ f := &model.KnowledgeFilter{CodeLangs: []string{"python"}}+ if !passesSearchFilter(code, f) {+ t.Fatal("matching codeLang must pass")+ }+ })++ t.Run("non-matching codeLang blocks", func(t *testing.T) {+ f := &model.KnowledgeFilter{CodeLangs: []string{"go"}}+ if passesSearchFilter(code, f) {+ t.Fatal("non-matching codeLang must block")+ }+ })++ t.Run("manual=true filter passes manual doc", func(t *testing.T) {+ f := &model.KnowledgeFilter{Manual: ptr(true)}+ if !passesSearchFilter(answer, f) {+ t.Fatal("manual=true filter must pass manual doc")+ }+ })++ t.Run("manual=true filter blocks non-manual doc", func(t *testing.T) {+ f := &model.KnowledgeFilter{Manual: ptr(true)}+ if passesSearchFilter(guide, f) {+ t.Fatal("manual=true filter must block non-manual doc")+ }+ })++ t.Run("manual=false filter passes non-manual doc", func(t *testing.T) {+ f := &model.KnowledgeFilter{Manual: ptr(false)}+ if !passesSearchFilter(guide, f) {+ t.Fatal("manual=false filter must pass non-manual doc")+ }+ })++ t.Run("manual=false filter blocks manual doc", func(t *testing.T) {+ f := &model.KnowledgeFilter{Manual: ptr(false)}+ if passesSearchFilter(answer, f) {+ t.Fatal("manual=false filter must block manual doc")+ }+ })++ t.Run("flowID match passes", func(t *testing.T) {+ f := &model.KnowledgeFilter{FlowID: ptr(int64(10))}+ if !passesSearchFilter(guide, f) {+ t.Fatal("matching flowID must pass")+ }+ })++ t.Run("flowID mismatch blocks", func(t *testing.T) {+ f := &model.KnowledgeFilter{FlowID: ptr(int64(99))}+ if passesSearchFilter(guide, f) {+ t.Fatal("non-matching flowID must block")+ }+ })++ t.Run("flowID filter blocks doc without flowID", func(t *testing.T) {+ f := &model.KnowledgeFilter{FlowID: ptr(int64(10))}+ if passesSearchFilter(answer, f) {+ t.Fatal("doc without flowID must be blocked when flowID filter is set")+ }+ })++ t.Run("combined filters all match", func(t *testing.T) {+ f := &model.KnowledgeFilter{+ DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide},+ GuideTypes: []model.KnowledgeGuideType{model.KnowledgeGuideTypePentest},+ FlowID: ptr(int64(10)),+ }+ if !passesSearchFilter(guide, f) {+ t.Fatal("all-matching combined filter must pass")+ }+ })++ t.Run("combined filters partial mismatch blocks", func(t *testing.T) {+ f := &model.KnowledgeFilter{+ DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide},+ FlowID: ptr(int64(99)), // wrong flowID+ }+ if passesSearchFilter(guide, f) {+ t.Fatal("partial mismatch in combined filter must block")+ }+ })+}++// ============================================================================+// Tests: SearchDocuments / SearchUserDocuments+// ============================================================================++func TestSearchDocuments(t *testing.T) {+ ctx := context.Background()++ t.Run("embedder unavailable returns error", func(t *testing.T) {+ ks := &knowledgeStore{embedder: &mockEmbedder{available: false}}+ _, err := ks.SearchDocuments(ctx, "query", nil, 5)+ if err == nil {+ t.Fatal("expected error when embedder unavailable")+ }+ })++ t.Run("nil embedder returns error", func(t *testing.T) {+ ks := &knowledgeStore{embedder: nil}+ _, err := ks.SearchDocuments(ctx, "query", nil, 5)+ if err == nil {+ t.Fatal("expected error when embedder is nil")+ }+ })++ t.Run("EmbedDocuments error propagates", func(t *testing.T) {+ ks := &knowledgeStore{+ embedder: &mockEmbedder{+ available: true,+ embedDocumentsFn: func(_ context.Context, _ []string) ([][]float32, error) {+ return nil, errors.New("embed error")+ },+ },+ }+ _, err := ks.SearchDocuments(ctx, "query", nil, 5)+ if err == nil {+ t.Fatal("expected error from EmbedDocuments")+ }+ })++ t.Run("embedder returning empty vectors is an error", func(t *testing.T) {+ ks := &knowledgeStore{+ embedder: &mockEmbedder{+ available: true,+ embedDocumentsFn: func(_ context.Context, _ []string) ([][]float32, error) {+ return [][]float32{}, nil+ },+ },+ }+ _, err := ks.SearchDocuments(ctx, "query", nil, 5)+ if err == nil {+ t.Fatal("expected error for empty vectors")+ }+ })++ t.Run("returns results with correct IDs and scores", func(t *testing.T) {+ db := &mockDB{+ searchKnowledge: func(_ context.Context, _ database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {+ return []database.SearchKnowledgeDocumentsRow{+ makeSearchRow("uuid-1", "content1", `{"doc_type":"answer"}`, 0.95),+ makeSearchRow("uuid-2", "content2", `{"doc_type":"guide"}`, 0.80),+ }, nil+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ results, err := ks.SearchDocuments(ctx, "test query", nil, 5)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != 2 {+ t.Fatalf("want 2 results, got %d", len(results))+ }+ // Critical: IDs must be populated (this was the original bug)+ if results[0].Document.ID != "uuid-1" {+ t.Fatalf("ID mismatch: want uuid-1, got %q", results[0].Document.ID)+ }+ if results[1].Document.ID != "uuid-2" {+ t.Fatalf("ID mismatch: want uuid-2, got %q", results[1].Document.ID)+ }+ if results[0].Score != 0.95 {+ t.Fatalf("score mismatch: want 0.95, got %f", results[0].Score)+ }+ if results[1].Score != 0.80 {+ t.Fatalf("score mismatch: want 0.80, got %f", results[1].Score)+ }+ })++ t.Run("correct params passed to db (threshold, limit, vector literal)", func(t *testing.T) {+ var gotParams database.SearchKnowledgeDocumentsParams+ db := &mockDB{+ searchKnowledge: func(_ context.Context, arg database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {+ gotParams = arg+ return nil, nil+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ _, err := ks.SearchDocuments(ctx, "q", nil, 7)+ if err != nil {+ t.Fatal(err)+ }+ if gotParams.MaxDistance != float64(1.0-defaultSearchThreshold) {+ t.Fatalf("distance threshold: want %f, got %f", float64(1.0-defaultSearchThreshold), gotParams.MaxDistance)+ }+ if gotParams.Lim != 7 {+ t.Fatalf("limit: want 7, got %d", gotParams.Lim)+ }+ vec, ok := gotParams.Embedding.(string)+ if !ok || len(vec) < 2 || vec[0] != '[' {+ t.Fatalf("vector literal format wrong: %v", gotParams.Embedding)+ }+ })++ t.Run("default limit applied when limit<=0", func(t *testing.T) {+ var gotLimit int32+ db := &mockDB{+ searchKnowledge: func(_ context.Context, arg database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {+ gotLimit = arg.Lim+ return nil, nil+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ _, err := ks.SearchDocuments(ctx, "q", nil, 0)+ if err != nil {+ t.Fatal(err)+ }+ if gotLimit != int32(defaultSearchLimit) {+ t.Fatalf("default limit: want %d, got %d", defaultSearchLimit, gotLimit)+ }+ })++ t.Run("go filter by docType applied after db fetch", func(t *testing.T) {+ db := &mockDB{+ searchKnowledge: func(_ context.Context, _ database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {+ return []database.SearchKnowledgeDocumentsRow{+ makeSearchRow("g1", "", `{"doc_type":"guide"}`, 0.9),+ makeSearchRow("a1", "", `{"doc_type":"answer"}`, 0.8),+ }, nil+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ results, err := ks.SearchDocuments(ctx, "q",+ &model.KnowledgeFilter{DocTypes: []model.KnowledgeDocType{model.KnowledgeDocTypeGuide}}, 10)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != 1 || results[0].Document.ID != "g1" {+ t.Fatalf("go filter should have kept only guide doc, got %d results", len(results))+ }+ })++ t.Run("go filter by flowID applied after db fetch", func(t *testing.T) {+ db := &mockDB{+ searchKnowledge: func(_ context.Context, _ database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {+ return []database.SearchKnowledgeDocumentsRow{+ makeSearchRow("f10", "", `{"doc_type":"answer","flow_id":10}`, 0.9),+ makeSearchRow("f20", "", `{"doc_type":"answer","flow_id":20}`, 0.8),+ }, nil+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ results, err := ks.SearchDocuments(ctx, "q",+ &model.KnowledgeFilter{FlowID: ptr(int64(10))}, 10)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != 1 || results[0].Document.ID != "f10" {+ t.Fatalf("flowID filter: expected f10, got %d results", len(results))+ }+ })++ t.Run("db error propagates", func(t *testing.T) {+ db := &mockDB{+ searchKnowledge: func(_ context.Context, _ database.SearchKnowledgeDocumentsParams) ([]database.SearchKnowledgeDocumentsRow, error) {+ return nil, errors.New("db error")+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ _, err := ks.SearchDocuments(ctx, "q", nil, 5)+ if err == nil {+ t.Fatal("expected error from db")+ }+ })+}++func TestSearchUserDocuments(t *testing.T) {+ ctx := context.Background()+ const userID = int64(42)++ t.Run("calls SearchUserKnowledgeDocuments with correct userID", func(t *testing.T) {+ var gotParams database.SearchUserKnowledgeDocumentsParams+ db := &mockDB{+ searchUserKnowledge: func(_ context.Context, arg database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {+ gotParams = arg+ return []database.SearchUserKnowledgeDocumentsRow{+ makeUserSearchRow("uuid-u1", "content", `{"doc_type":"answer","user_id":42}`, 0.88),+ }, nil+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}++ results, err := ks.SearchUserDocuments(ctx, userID, "query", nil, 3)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != 1 {+ t.Fatalf("want 1 result, got %d", len(results))+ }+ if results[0].Document.ID != "uuid-u1" {+ t.Fatalf("ID mismatch: want uuid-u1, got %q", results[0].Document.ID)+ }+ if results[0].Score != 0.88 {+ t.Fatalf("score mismatch: want 0.88, got %f", results[0].Score)+ }+ if gotParams.UserID.String != "42" {+ t.Fatalf("userID param: want 42, got %q", gotParams.UserID.String)+ }+ if gotParams.Lim != 3 {+ t.Fatalf("limit: want 3, got %d", gotParams.Lim)+ }+ })++ // SECURITY: the user_id must always be sent to the database layer so the+ // database can enforce ownership — it must never be omitted or zero.+ t.Run("SECURITY: userID is always sent to DB", func(t *testing.T) {+ var capturedUserID string+ db := &mockDB{+ searchUserKnowledge: func(_ context.Context, arg database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {+ capturedUserID = arg.UserID.String+ return nil, nil+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ _, err := ks.SearchUserDocuments(ctx, userID, "q", nil, 5)+ if err != nil {+ t.Fatal(err)+ }+ if capturedUserID != "42" {+ t.Fatalf("SECURITY: user_id must always be passed to DB, got %q", capturedUserID)+ }+ })++ t.Run("go filter applied for user search", func(t *testing.T) {+ db := &mockDB{+ searchUserKnowledge: func(_ context.Context, _ database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {+ return []database.SearchUserKnowledgeDocumentsRow{+ makeUserSearchRow("c1", "", `{"doc_type":"code","code_lang":"go"}`, 0.9),+ makeUserSearchRow("c2", "", `{"doc_type":"code","code_lang":"python"}`, 0.7),+ }, nil+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ results, err := ks.SearchUserDocuments(ctx, userID, "q",+ &model.KnowledgeFilter{CodeLangs: []string{"go"}}, 10)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != 1 || results[0].Document.ID != "c1" {+ t.Fatalf("codeLang filter: expected c1 only, got %d results", len(results))+ }+ })++ t.Run("db error propagates", func(t *testing.T) {+ db := &mockDB{+ searchUserKnowledge: func(_ context.Context, _ database.SearchUserKnowledgeDocumentsParams) ([]database.SearchUserKnowledgeDocumentsRow, error) {+ return nil, errors.New("db error")+ },+ }+ ks := &knowledgeStore{db: db, embedder: &mockEmbedder{available: true}}+ _, err := ks.SearchUserDocuments(ctx, userID, "q", nil, 5)+ if err == nil {+ t.Fatal("expected error from db")}})}@@ -856,8 +1253,8 @@ func TestGetUserDocument(t *testing.T) {if gotParams.Uuid != "doc-uuid" {t.Fatalf("uuid mismatch: %q", gotParams.Uuid)}- if gotParams.Cmetadata.String != "7" {- t.Fatalf("userID param mismatch: %q", gotParams.Cmetadata.String)+ if gotParams.UserID.String != "7" {+ t.Fatalf("userID param mismatch: %q", gotParams.UserID.String)}if doc.ID != "doc-uuid" {t.Fatal("document ID mismatch")@@ -976,11 +1373,11 @@ func TestDeleteUserDocument(t *testing.T) {t.Fatal(err)}// Ownership enforced: correct uuid and userID sent to DB- if gotDelParams.Column1.String != "my-doc" {- t.Fatalf("uuid param: want my-doc, got %q", gotDelParams.Column1.String)+ if gotDelParams.Uuid.String != "my-doc" {+ t.Fatalf("uuid param: want my-doc, got %q", gotDelParams.Uuid.String)}- if gotDelParams.Column2.String != "3" {- t.Fatalf("user_id param: want 3, got %q", gotDelParams.Column2.String)+ if gotDelParams.UserID.String != "3" {+ t.Fatalf("user_id param: want 3, got %q", gotDelParams.UserID.String)}if len(pub.deletedDocs) != 1 {t.Fatal("event not published")@@ -1204,8 +1601,8 @@ func TestUpdateDocument(t *testing.T) {},updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {return database.UpdateKnowledgeDocumentRow{- ID: arg.Column1.String,- Document: arg.Column3.String,+ ID: arg.Uuid.String,+ Document: arg.Document.String,Cmetadata: sql.NullString{String: existingMeta, Valid: true},}, nil},@@ -1296,8 +1693,8 @@ func TestUpdateDocument(t *testing.T) {updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {gotParams = argreturn database.UpdateKnowledgeDocumentRow{- ID: arg.Column1.String,- Document: arg.Column3.String,+ ID: arg.Uuid.String,+ Document: arg.Document.String,Cmetadata: sql.NullString{String: `{"doc_type":"guide","guide_type":"pentest","question":"new q"}`,Valid: true,@@ -1320,16 +1717,16 @@ func TestUpdateDocument(t *testing.T) {}// Correct uuid passed to update- if gotParams.Column1.String != "doc-id" {- t.Fatalf("uuid mismatch: %q", gotParams.Column1.String)+ if gotParams.Uuid.String != "doc-id" {+ t.Fatalf("uuid mismatch: %q", gotParams.Uuid.String)}// New content passed- if gotParams.Column3.String != "new content" {- t.Fatalf("document mismatch: %q", gotParams.Column3.String)+ if gotParams.Document.String != "new content" {+ t.Fatalf("document mismatch: %q", gotParams.Document.String)}// Embedding formatted as vector literal- if gotParams.Column2 == "" || gotParams.Column2[0] != '[' {- t.Fatalf("embedding format wrong: %q", gotParams.Column2)+ if gotParams.Embedding == "" || gotParams.Embedding[0] != '[' {+ t.Fatalf("embedding format wrong: %q", gotParams.Embedding)}// Updated doc returned@@ -1359,12 +1756,12 @@ func TestUpdateDocumentDocTypeChange(t *testing.T) {return makeRow(uuid, "old", initialMeta), nil},updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {- *stored = string(arg.Column4.RawMessage)+ *stored = string(arg.Cmetadata.RawMessage)return database.UpdateKnowledgeDocumentRow{- ID: arg.Column1.String,- Document: arg.Column3.String,+ ID: arg.Uuid.String,+ Document: arg.Document.String,Cmetadata: sql.NullString{- String: string(arg.Column4.RawMessage),+ String: string(arg.Cmetadata.RawMessage),Valid: true,},}, nil@@ -1578,12 +1975,12 @@ func TestUpdateDocumentSizeCalculation(t *testing.T) {return row, nil},updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {- *stored = string(arg.Column4.RawMessage)+ *stored = string(arg.Cmetadata.RawMessage)return database.UpdateKnowledgeDocumentRow{- ID: arg.Column1.String,- Document: arg.Column3.String,+ ID: arg.Uuid.String,+ Document: arg.Document.String,Cmetadata: sql.NullString{- String: string(arg.Column4.RawMessage),+ String: string(arg.Cmetadata.RawMessage),Valid: true,},}, nil@@ -1702,12 +2099,12 @@ func TestUpdateDocumentPreservesOriginalOwner(t *testing.T) {return makeRow(uuid, "old", originalMeta), nil},updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {- storedMeta = string(arg.Column4.RawMessage)+ storedMeta = string(arg.Cmetadata.RawMessage)return database.UpdateKnowledgeDocumentRow{- ID: arg.Column1.String,- Document: arg.Column3.String,+ ID: arg.Uuid.String,+ Document: arg.Document.String,Cmetadata: sql.NullString{- String: string(arg.Column4.RawMessage),+ String: string(arg.Cmetadata.RawMessage),Valid: true,},}, nil@@ -1755,7 +2152,7 @@ func TestUpdateUserDocument(t *testing.T) {db := &mockDB{getUserKnowledge: func(_ context.Context, arg database.GetUserKnowledgeDocumentParams) (database.GetUserKnowledgeDocumentRow, error) {fetchedUUID = arg.Uuid- fetchedUserID = arg.Cmetadata.String+ fetchedUserID = arg.UserID.Stringreturn database.GetUserKnowledgeDocumentRow{ID: arg.Uuid,Document: "old",@@ -1764,8 +2161,8 @@ func TestUpdateUserDocument(t *testing.T) {},updateKnowledge: func(_ context.Context, arg database.UpdateKnowledgeDocumentParams) (database.UpdateKnowledgeDocumentRow, error) {return database.UpdateKnowledgeDocumentRow{- ID: arg.Column1.String,- Document: arg.Column3.String,+ ID: arg.Uuid.String,+ Document: arg.Document.String,Cmetadata: sql.NullString{String: existingMeta,Valid: true,diff --git a/backend/pkg/database/querier.go b/backend/pkg/database/querier.goindex 0cfa6e73..6f6335b1 100644--- a/backend/pkg/database/querier.go+++ b/backend/pkg/database/querier.go@@ -40,12 +40,12 @@ type Querier interface {DeleteFlowAssistantLog(ctx context.Context, id int64) error// Delete all memory-type documents for a specific flow.// Called on flow deletion to free long-term memory that will never be re-used.- // $1 is the decimal text representation of the flow ID (e.g. "55"), matching the+ // flow_id is the decimal text representation of the flow ID (e.g. "55"), matching the// text result of (cmetadata ->> 'flow_id') which uses JSON ->> extraction.- DeleteFlowMemoryDocuments(ctx context.Context, dollar_1 sql.NullString) error+ DeleteFlowMemoryDocuments(ctx context.Context, flowID sql.NullString) errorDeleteFlowTemplate(ctx context.Context, arg DeleteFlowTemplateParams) error// Delete a knowledge document by UUID (admin — no user_id check).- DeleteKnowledgeDocument(ctx context.Context, dollar_1 sql.NullString) error+ DeleteKnowledgeDocument(ctx context.Context, uuid sql.NullString) errorDeletePrompt(ctx context.Context, id int64) errorDeleteProvider(ctx context.Context, id int64) (Provider, error)DeleteSubtask(ctx context.Context, id int64) error@@ -238,9 +238,23 @@ type Querier interface {// List all knowledge documents excluding the noisy memory type (admin view).ListAllKnowledgeDocuments(ctx context.Context) ([]ListAllKnowledgeDocumentsRow, error)// List non-memory knowledge documents belonging to a specific flow (admin scoped).- ListFlowKnowledgeDocuments(ctx context.Context, cmetadata sql.NullString) ([]ListFlowKnowledgeDocumentsRow, error)+ ListFlowKnowledgeDocuments(ctx context.Context, flowID sql.NullString) ([]ListFlowKnowledgeDocumentsRow, error)// List all non-memory knowledge documents owned by a specific user (user-scoped view).- ListUserKnowledgeDocuments(ctx context.Context, cmetadata sql.NullString) ([]ListUserKnowledgeDocumentsRow, error)+ ListUserKnowledgeDocuments(ctx context.Context, userID sql.NullString) ([]ListUserKnowledgeDocumentsRow, error)+ // Vector similarity search over all knowledge documents (admin view, no user filter).+ // Returns rows ordered by cosine similarity descending (highest score first).+ // embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'+ // max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold),+ // e.g. pass 0.8 to get documents with similarity score > 0.2+ // lim maximum number of rows to return+ SearchKnowledgeDocuments(ctx context.Context, arg SearchKnowledgeDocumentsParams) ([]SearchKnowledgeDocumentsRow, error)+ // Vector similarity search scoped to a specific user (by cmetadata user_id).+ // Returns rows ordered by cosine similarity descending (highest score first).+ // embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'+ // max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold)+ // lim maximum number of rows to return+ // user_id owner filter as a decimal text string (e.g. "42")+ SearchUserKnowledgeDocuments(ctx context.Context, arg SearchUserKnowledgeDocumentsParams) ([]SearchUserKnowledgeDocumentsRow, error)UpdateAPIToken(ctx context.Context, arg UpdateAPITokenParams) (ApiToken, error)UpdateAssistant(ctx context.Context, arg UpdateAssistantParams) (Assistant, error)UpdateAssistantLanguage(ctx context.Context, arg UpdateAssistantLanguageParams) (Assistant, error)@@ -263,8 +277,8 @@ type Querier interface {UpdateFlowTitle(ctx context.Context, arg UpdateFlowTitleParams) (Flow, error)UpdateFlowToolCallIDTemplate(ctx context.Context, arg UpdateFlowToolCallIDTemplateParams) (Flow, error)// Update an existing document's embedding, text and metadata atomically.- // $2 must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'- // $4 must be valid JSON text.+ // embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'+ // cmetadata must be valid JSON text.UpdateKnowledgeDocument(ctx context.Context, arg UpdateKnowledgeDocumentParams) (UpdateKnowledgeDocumentRow, error)UpdateMsgChain(ctx context.Context, arg UpdateMsgChainParams) (Msgchain, error)UpdateMsgChainUsage(ctx context.Context, arg UpdateMsgChainUsageParams) (Msgchain, error)diff --git a/backend/sqlc/models/knowledge.sql b/backend/sqlc/models/knowledge.sqlindex 1c859dc0..a0812409 100644--- a/backend/sqlc/models/knowledge.sql+++ b/backend/sqlc/models/knowledge.sql@@ -7,7 +7,7 @@ SELECTFROM langchain_pg_embedding eINNER JOIN langchain_pg_collection c ON e.collection_id = c.uuidWHERE c.name = 'langchain'- AND e.uuid::text = $1;+ AND e.uuid::text = sqlc.arg(uuid);-- name: GetUserKnowledgeDocument :one-- Fetch a single knowledge document by UUID, scoped to a specific user.@@ -18,8 +18,8 @@ SELECTFROM langchain_pg_embedding eINNER JOIN langchain_pg_collection c ON e.collection_id = c.uuidWHERE c.name = 'langchain'- AND e.uuid::text = $1- AND (e.cmetadata ->> 'user_id') = $2;+ AND e.uuid::text = sqlc.arg(uuid)+ AND (e.cmetadata ->> 'user_id') = sqlc.arg(user_id);-- name: ListAllKnowledgeDocuments :many-- List all knowledge documents excluding the noisy memory type (admin view).@@ -43,7 +43,7 @@ FROM langchain_pg_embedding eINNER JOIN langchain_pg_collection c ON e.collection_id = c.uuidWHERE c.name = 'langchain'AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')- AND (e.cmetadata ->> 'flow_id') = $1+ AND (e.cmetadata ->> 'flow_id') = sqlc.arg(flow_id)ORDER BY e.uuid;-- name: ListUserKnowledgeDocuments :many@@ -56,19 +56,19 @@ FROM langchain_pg_embedding eINNER JOIN langchain_pg_collection c ON e.collection_id = c.uuidWHERE c.name = 'langchain'AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')- AND (e.cmetadata ->> 'user_id') = $1+ AND (e.cmetadata ->> 'user_id') = sqlc.arg(user_id)ORDER BY e.uuid;-- name: UpdateKnowledgeDocument :one-- Update an existing document's embedding, text and metadata atomically.--- $2 must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'--- $4 must be valid JSON text.+-- embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'+-- cmetadata must be valid JSON text.UPDATE langchain_pg_embeddingSET- embedding = $2::vector,- document = $3,- cmetadata = $4::json-WHERE uuid::text = $1+ embedding = sqlc.arg(embedding)::vector,+ document = sqlc.arg(document),+ cmetadata = sqlc.arg(cmetadata)::json+WHERE uuid::text = sqlc.arg(uuid)AND collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain')RETURNINGuuid::text AS id,@@ -78,22 +78,63 @@ RETURNING-- name: DeleteKnowledgeDocument :exec-- Delete a knowledge document by UUID (admin — no user_id check).DELETE FROM langchain_pg_embedding-WHERE uuid::text = $1+WHERE uuid::text = sqlc.arg(uuid)AND collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain');-- name: DeleteUserKnowledgeDocument :exec-- Delete a knowledge document by UUID, only if it belongs to the given user.DELETE FROM langchain_pg_embedding-WHERE uuid::text = $1- AND (cmetadata ->> 'user_id') = $2+WHERE uuid::text = sqlc.arg(uuid)+ AND (cmetadata ->> 'user_id') = sqlc.arg(user_id)AND collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain');+-- name: SearchKnowledgeDocuments :many+-- Vector similarity search over all knowledge documents (admin view, no user filter).+-- Returns rows ordered by cosine similarity descending (highest score first).+-- embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'+-- max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold),+-- e.g. pass 0.8 to get documents with similarity score > 0.2+-- lim maximum number of rows to return+SELECT+ e.uuid::text AS id,+ COALESCE(e.document, '') AS document,+ COALESCE(e.cmetadata::text, '{}') AS cmetadata,+ (1.0 - (e.embedding <=> sqlc.arg(embedding)::vector))::float8 AS score+FROM langchain_pg_embedding e+INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid+WHERE c.name = 'langchain'+ AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')+ AND (e.embedding <=> sqlc.arg(embedding)::vector)::float8 < sqlc.arg(max_distance)::float8+ORDER BY e.embedding <=> sqlc.arg(embedding)::vector+LIMIT sqlc.arg(lim)::int;++-- name: SearchUserKnowledgeDocuments :many+-- Vector similarity search scoped to a specific user (by cmetadata user_id).+-- Returns rows ordered by cosine similarity descending (highest score first).+-- embedding query vector as a PostgreSQL vector literal, e.g. '[0.1,0.2,...]'+-- max_distance cosine-distance upper bound (exclusive); equals (1 - score_threshold)+-- lim maximum number of rows to return+-- user_id owner filter as a decimal text string (e.g. "42")+SELECT+ e.uuid::text AS id,+ COALESCE(e.document, '') AS document,+ COALESCE(e.cmetadata::text, '{}') AS cmetadata,+ (1.0 - (e.embedding <=> sqlc.arg(embedding)::vector))::float8 AS score+FROM langchain_pg_embedding e+INNER JOIN langchain_pg_collection c ON e.collection_id = c.uuid+WHERE c.name = 'langchain'+ AND COALESCE(e.cmetadata ->> 'doc_type', '') NOT IN ('memory')+ AND (e.cmetadata ->> 'user_id') = sqlc.arg(user_id)+ AND (e.embedding <=> sqlc.arg(embedding)::vector)::float8 < sqlc.arg(max_distance)::float8+ORDER BY e.embedding <=> sqlc.arg(embedding)::vector+LIMIT sqlc.arg(lim)::int;+-- name: DeleteFlowMemoryDocuments :exec-- Delete all memory-type documents for a specific flow.-- Called on flow deletion to free long-term memory that will never be re-used.--- $1 is the decimal text representation of the flow ID (e.g. "55"), matching the+-- flow_id is the decimal text representation of the flow ID (e.g. "55"), matching the-- text result of (cmetadata ->> 'flow_id') which uses JSON ->> extraction.DELETE FROM langchain_pg_embeddingWHERE collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain')AND COALESCE(cmetadata ->> 'doc_type', '') = 'memory'- AND (cmetadata ->> 'flow_id') = $1;+ AND (cmetadata ->> 'flow_id') = sqlc.arg(flow_id);