Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
feat(config): add new embedding and rename database connection pool settings
Dmitry Ng39f122467d3d
diff --git a/.env.example b/.env.exampleindex 46a0a564..4b95dc8f 100644--- a/.env.example+++ b/.env.example@@ -73,6 +73,7 @@ EMBEDDING_KEY=EMBEDDING_MODEL=EMBEDDING_PROVIDER=EMBEDDING_BATCH_SIZE=+EMBEDDING_MAX_TEXT_BYTES=EMBEDDING_STRIP_NEW_LINES=## Summarizer@@ -226,6 +227,11 @@ PENTAGI_POSTGRES_USER=postgresPENTAGI_POSTGRES_PASSWORD=postgres # change this to improve securityPENTAGI_POSTGRES_DB=pentagidb+## Postgres (pgvector) connection pool settings+DATABASE_MAX_OPEN_CONNS=+DATABASE_MAX_IDLE_CONNS=+DATABASE_VECTOR_MAX_CONNS=+## Graphiti knowledge graph settings## Set GRAPHITI_ENABLED=true and GRAPHITI_URL=http://graphiti:8000 to enable embedded GraphitiGRAPHITI_ENABLED=falsediff --git a/README.md b/README.mdindex 5f7ef98d..485bc701 100644--- a/README.md+++ b/README.md@@ -2619,23 +2619,23 @@ PentAGI opens two independent connection pools to the same Postgres instance:| Pool | Env var | Default | Used by ||---|---|---|---|-| Shared `sql.DB` | `DB_MAX_OPEN_CONNS` | `25` | All sqlc queries and GORM handlers share a single `*sql.DB` |-| Shared `pgxpool` | `DB_VECTOR_MAX_CONNS` | `10` | All pgvector stores (agent memory + knowledge API) share a single pool |+| Shared `sql.DB` | `DATABASE_MAX_OPEN_CONNS` | `25` | All sqlc queries and GORM handlers share a single `*sql.DB` |+| Shared `pgxpool` | `DATABASE_VECTOR_MAX_CONNS` | `10` | All pgvector stores (agent memory + knowledge API) share a single pool |Additional tuning knob:-- `DB_MAX_IDLE_CONNS` — maximum idle connections kept open in the `sql.DB` pool between requests (default: `5`).+- `DATABASE_MAX_IDLE_CONNS` — maximum idle connections kept open in the `sql.DB` pool between requests (default: `5`).**Budget for the stock `vxcontrol/pgvector` image** (`max_connections = 100`, `superuser_reserved_connections = 3`):```Available for client connections = 97- pentagi sql.DB (DB_MAX_OPEN_CONNS) = 25- pentagi pgxpool (DB_VECTOR_MAX_CONNS) = 10- pgexporter = 3- autovacuum workers = 3+ pentagi sql.DB (DATABASE_MAX_OPEN_CONNS) = 25+ pentagi pgxpool (DATABASE_VECTOR_MAX_CONNS) = 10+ pgexporter = 3+ autovacuum workers = 3─────────────────────────────────────────- Total consumed = 41- Free buffer = 56 (≈ 58 %)+ Total consumed = 41+ Free buffer = 56 (≈ 58 %)```The defaults are sized for **10 parallel flows** with concurrent API requests. If you run more flows or deploy multiple PentAGI instances against the same Postgres, raise `max_connections` via the `command` override in `docker-compose.yml` and increase the pool sizes proportionally:@@ -3084,6 +3084,7 @@ EMBEDDING_URL= # Optional custom API endpointEMBEDDING_KEY= # API key for the provider (if required)EMBEDDING_BATCH_SIZE=100 # Number of documents to process in a batchEMBEDDING_STRIP_NEW_LINES=true # Whether to remove new lines from text before embedding+EMBEDDING_MAX_TEXT_BYTES=8192 # Max bytes of text sent to embedding model per document (byte proxy for token limit)# Advanced settingsPROXY_URL= # Optional proxy for all API callsdiff --git a/backend/cmd/ftester/worker/executor.go b/backend/cmd/ftester/worker/executor.goindex 2cdb5dcf..2d2f6b6a 100644--- a/backend/cmd/ftester/worker/executor.go+++ b/backend/cmd/ftester/worker/executor.go@@ -46,6 +46,7 @@ type toolExecutor struct {dockerClient docker.DockerClienthandlers providers.FlowProviderHandlersstore *pgvector.Store+ embedder embeddings.EmbeddergraphitiClient *graphiti.Clientproxies mocks.ProxyProvidersuserID int64@@ -103,6 +104,7 @@ func newToolExecutor(dockerClient: dockerClient,handlers: handlers,store: store,+ embedder: embedder,graphitiClient: graphitiClient,proxies: proxies,userID: userID,@@ -247,6 +249,9 @@ func (te *toolExecutor) GetTool(ctx context.Context, funcName string) (tools.Toote.subtaskID,te.replacer,te.store,+ te.embedder,+ te.db,+ te.cfg.EmbeddingMaxTextBytes,te.proxies.GetVectorStoreLogProvider(),te.proxies.GetKnowledgeProvider(),), nil@@ -259,6 +264,9 @@ func (te *toolExecutor) GetTool(ctx context.Context, funcName string) (tools.Toote.subtaskID,te.replacer,te.store,+ te.embedder,+ te.db,+ te.cfg.EmbeddingMaxTextBytes,te.proxies.GetVectorStoreLogProvider(),te.proxies.GetKnowledgeProvider(),), nil@@ -271,6 +279,9 @@ func (te *toolExecutor) GetTool(ctx context.Context, funcName string) (tools.Toote.subtaskID,te.replacer,te.store,+ te.embedder,+ te.db,+ te.cfg.EmbeddingMaxTextBytes,te.proxies.GetVectorStoreLogProvider(),te.proxies.GetKnowledgeProvider(),), nildiff --git a/backend/cmd/installer/wizard/controller/controller.go b/backend/cmd/installer/wizard/controller/controller.goindex 3080a632..fe98d179 100644--- a/backend/cmd/installer/wizard/controller/controller.go+++ b/backend/cmd/installer/wizard/controller/controller.go@@ -1197,6 +1197,7 @@ type EmbedderConfig struct {Model loader.EnvVar // EMBEDDING_MODELBatchSize loader.EnvVar // EMBEDDING_BATCH_SIZEStripNewLines loader.EnvVar // EMBEDDING_STRIP_NEW_LINES+ MaxTextBytes loader.EnvVar // EMBEDDING_MAX_TEXT_BYTES// computed fields (not directly mapped to env vars)Configured bool@@ -1212,6 +1213,7 @@ func (c *controller) GetEmbedderConfig() *EmbedderConfig {config.Model, _ = c.GetVar("EMBEDDING_MODEL")config.BatchSize, _ = c.GetVar("EMBEDDING_BATCH_SIZE")config.StripNewLines, _ = c.GetVar("EMBEDDING_STRIP_NEW_LINES")+ config.MaxTextBytes, _ = c.GetVar("EMBEDDING_MAX_TEXT_BYTES")config.Installed = c.checker.PentagiInstalled// Determine if configured based on provider requirements@@ -1259,6 +1261,9 @@ func (c *controller) UpdateEmbedderConfig(config *EmbedderConfig) error {if err := c.SetVar("EMBEDDING_STRIP_NEW_LINES", config.StripNewLines.Value); err != nil {return fmt.Errorf("failed to set EMBEDDING_STRIP_NEW_LINES: %w", err)}+ if err := c.SetVar("EMBEDDING_MAX_TEXT_BYTES", config.MaxTextBytes.Value); err != nil {+ return fmt.Errorf("failed to set EMBEDDING_MAX_TEXT_BYTES: %w", err)+ }return nil}@@ -1271,6 +1276,7 @@ func (c *controller) ResetEmbedderConfig() *EmbedderConfig {"EMBEDDING_MODEL","EMBEDDING_BATCH_SIZE","EMBEDDING_STRIP_NEW_LINES",+ "EMBEDDING_MAX_TEXT_BYTES",}if err := c.ResetVars(vars); err != nil {@@ -2215,6 +2221,7 @@ func (c *controller) getVariableDescription(varName string) string {"EMBEDDING_MODEL": locale.EnvDesc_EMBEDDING_MODEL,"EMBEDDING_BATCH_SIZE": locale.EnvDesc_EMBEDDING_BATCH_SIZE,"EMBEDDING_STRIP_NEW_LINES": locale.EnvDesc_EMBEDDING_STRIP_NEW_LINES,+ "EMBEDDING_MAX_TEXT_BYTES": locale.EnvDesc_EMBEDDING_MAX_TEXT_BYTES,"ASK_USER": locale.EnvDesc_ASK_USER,@@ -2455,6 +2462,7 @@ var criticalVariables = map[string]bool{"EMBEDDING_MODEL": true,"EMBEDDING_BATCH_SIZE": true,"EMBEDDING_STRIP_NEW_LINES": true,+ "EMBEDDING_MAX_TEXT_BYTES": true,// Docker configuration changes"DOCKER_INSIDE": true,diff --git a/backend/cmd/installer/wizard/locale/locale.go b/backend/cmd/installer/wizard/locale/locale.goindex 372572a4..5ae20f77 100644--- a/backend/cmd/installer/wizard/locale/locale.go+++ b/backend/cmd/installer/wizard/locale/locale.go@@ -1662,6 +1662,9 @@ Choose carefully as changing providers requires reindexing all stored data.`EmbedderFormStripNewLines = "Strip New Lines"EmbedderFormStripNewLinesDesc = "Remove line breaks from text before embedding (true/false)"+ EmbedderFormMaxTextBytes = "Max Text Bytes"+ EmbedderFormMaxTextBytesDesc = "Maximum number of bytes per text chunk sent to the embedding API (e.g. 8192)"+EmbedderFormHelpTitle = "Embedding Configuration"EmbedderFormHelpContent = `Configure text vectorization for semantic search and knowledge storage.@@ -2242,6 +2245,7 @@ const (EnvDesc_EMBEDDING_MODEL = "Embedding Model"EnvDesc_EMBEDDING_BATCH_SIZE = "Embedding Batch Size"EnvDesc_EMBEDDING_STRIP_NEW_LINES = "Embedding Strip New Lines"+ EnvDesc_EMBEDDING_MAX_TEXT_BYTES = "Embedding Max Text Bytes"EnvDesc_ASK_USER = "Human-in-the-loop"diff --git a/backend/cmd/installer/wizard/models/embedder_form.go b/backend/cmd/installer/wizard/models/embedder_form.goindex ba3f8324..784a0aad 100644--- a/backend/cmd/installer/wizard/models/embedder_form.go+++ b/backend/cmd/installer/wizard/models/embedder_form.go@@ -262,6 +262,11 @@ func (m *EmbedderFormModel) BuildForm() tea.Cmd {fields = append(fields, m.createStripNewLinesField(config))}+ // Max text bytes field (always show except for disabled)+ if providerInfo.ID != locale.EmbedderProviderIDDisabled {+ fields = append(fields, m.createMaxTextBytesField(config))+ }+m.SetFormFields(fields)return nil}@@ -352,6 +357,20 @@ func (m *EmbedderFormModel) createStripNewLinesField(config *controller.Embedder}}+func (m *EmbedderFormModel) createMaxTextBytesField(config *controller.EmbedderConfig) FormField {+ input := NewTextInput(m.GetStyles(), m.GetWindow(), config.MaxTextBytes)++ return FormField{+ Key: "max_text_bytes",+ Title: locale.EmbedderFormMaxTextBytes,+ Description: locale.EmbedderFormMaxTextBytesDesc,+ Required: false,+ Masked: false,+ Input: input,+ Value: input.Value(),+ }+}+func (m *EmbedderFormModel) GetFormTitle() string {return locale.EmbedderFormTitle}@@ -432,6 +451,15 @@ func (m *EmbedderFormModel) GetCurrentConfiguration() string {locale.EmbedderFormStripNewLines, m.GetStyles().Info.Render(stripNewLines)))}+ maxTextBytes := config.MaxTextBytes.Value+ if maxTextBytes == "" {+ maxTextBytes = config.MaxTextBytes.Default+ }+ if maxTextBytes != "" {+ sections = append(sections, fmt.Sprintf("• %s: %s",+ locale.EmbedderFormMaxTextBytes, m.GetStyles().Info.Render(maxTextBytes)))+ }+return strings.Join(sections, "\n")}@@ -477,6 +505,7 @@ func (m *EmbedderFormModel) HandleSave() error {Model: config.Model,BatchSize: config.BatchSize,StripNewLines: config.StripNewLines,+ MaxTextBytes: config.MaxTextBytes,}// set provider@@ -514,6 +543,14 @@ func (m *EmbedderFormModel) HandleSave() error {return fmt.Errorf("invalid boolean value for strip newlines: %s (must be 'true' or 'false')", value)}newConfig.StripNewLines.Value = value+ case "max_text_bytes":+ // validate numeric input+ if value != "" {+ if intVal, err := strconv.Atoi(value); err != nil || intVal <= 0 {+ return fmt.Errorf("invalid max text bytes: %s (must be a positive number)", value)+ }+ }+ newConfig.MaxTextBytes.Value = value}}diff --git a/backend/cmd/pentagi/main.go b/backend/cmd/pentagi/main.goindex 1f1fec9c..1cebf233 100644--- a/backend/cmd/pentagi/main.go+++ b/backend/cmd/pentagi/main.go@@ -20,6 +20,7 @@ import ("pentagi/pkg/docker""pentagi/pkg/graph/subscriptions"obs "pentagi/pkg/observability"+ "pentagi/pkg/observability/profiling""pentagi/pkg/providers"router "pentagi/pkg/server""pentagi/pkg/version"@@ -124,6 +125,8 @@ func main() {log.Println("Database schema updated successfully")+ go profiling.Start()+client, err := docker.NewDockerClient(ctx, queries, cfg)if err != nil {log.Fatalf("Docker runtime client initialization failed: %v", err)diff --git a/backend/docs/config.md b/backend/docs/config.mdindex 42b90ab5..c53f41b0 100644--- a/backend/docs/config.md+++ b/backend/docs/config.md@@ -127,17 +127,17 @@ The environment variables documented below remain the source of truth for configThese settings control basic application behavior and are foundational for the system's operation.-| Option | Environment Variable | Default Value | Description |-| -------------- | -------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------ |-| DatabaseURL | `DATABASE_URL` | `postgres://pentagiuser:pentagipass@pgvector:5432/pentagidb?sslmode=disable` | Connection string for the PostgreSQL database with pgvector extension |-| DBMaxOpenConns | `DB_MAX_OPEN_CONNS` | `25` | Maximum open connections in the shared `sql.DB` pool (sqlc + GORM combined). See [database.md §Connection Pooling](database.md#connection-pooling). |-| DBMaxIdleConns | `DB_MAX_IDLE_CONNS` | `5` | Maximum idle connections kept open between requests |-| DBVectorMaxConns | `DB_VECTOR_MAX_CONNS` | `10` | Maximum connections in the shared `pgxpool` for all pgvector stores |-| Debug | `DEBUG` | `false` | Enables debug mode with additional logging |-| DataDir | `DATA_DIR` | `./data` | Directory for storing persistent data |-| AskUser | `ASK_USER` | `false` | When enabled, requires explicit user confirmation for certain operations |-| InstallationID | `INSTALLATION_ID` | *(none)* | Unique installation identifier for PentAGI Cloud API communication |-| LicenseKey | `LICENSE_KEY` | *(none)* | License key for PentAGI Cloud API authentication and feature activation |+| Option | Environment Variable | Default Value | Description |+| ---------------- | --------------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------ |+| DatabaseURL | `DATABASE_URL` | `postgres://pentagiuser:pentagipass@pgvector:5432/pentagidb?sslmode=disable` | Connection string for the PostgreSQL database with pgvector extension |+| DBMaxOpenConns | `DATABASE_MAX_OPEN_CONNS` | `25` | Maximum open connections in the shared `sql.DB` pool (sqlc + GORM combined). See [database.md §Connection Pooling](database.md#connection-pooling). |+| DBMaxIdleConns | `DATABASE_MAX_IDLE_CONNS` | `5` | Maximum idle connections kept open between requests |+| DBVectorMaxConns | `DATABASE_VECTOR_MAX_CONNS` | `10` | Maximum connections in the shared `pgxpool` for all pgvector stores |+| Debug | `DEBUG` | `false` | Enables debug mode with additional logging |+| DataDir | `DATA_DIR` | `./data` | Directory for storing persistent data |+| AskUser | `ASK_USER` | `false` | When enabled, requires explicit user confirmation for certain operations |+| InstallationID | `INSTALLATION_ID` | *(none)* | Unique installation identifier for PentAGI Cloud API communication |+| LicenseKey | `LICENSE_KEY` | *(none)* | License key for PentAGI Cloud API authentication and feature activation |### Usage Details@@ -870,6 +870,7 @@ These settings control the vector embedding service used for semantic search and| EmbeddingStripNewLines | `EMBEDDING_STRIP_NEW_LINES` | `true` | Whether to strip newlines before embedding (improves quality) || EmbeddingBatchSize | `EMBEDDING_BATCH_SIZE` | `512` | Batch size for embedding operations (affects memory usage and performance) || EmbeddingProvider | `EMBEDDING_PROVIDER` | `openai` | Provider for embeddings (openai, ollama, mistral, jina, huggingface) |+| EmbeddingMaxTextBytes | `EMBEDDING_MAX_TEXT_BYTES` | `8192` | Maximum byte size of text sent to the embedding model per document. Acts as a byte-level proxy for token limits (e.g. 8192 tokens for OpenAI models). When a stored document exceeds this limit the heavy content field (Guide/Answer/Code) is truncated to fit before computing the vector; the full original text is always preserved in the database. Reduce if your model has a smaller context window. |### Usage Detailsdiff --git a/backend/docs/database.md b/backend/docs/database.mdindex 1074eede..c4e17fdc 100644--- a/backend/docs/database.md+++ b/backend/docs/database.md@@ -1203,10 +1203,10 @@ PentAGI opens two independent connection pools to the same Postgres instance:| Pool | Env var | Default | Used by ||---|---|---|---|-| Shared `sql.DB` | `DB_MAX_OPEN_CONNS` | `25` | sqlc `Queries` and GORM — both clients are backed by the **same** `*sql.DB` created in `main.go` |-| Shared `pgxpool` | `DB_VECTOR_MAX_CONNS` | `10` | All `pgvector.Store` instances: every flow/assistant tool executor + knowledge API |+| Shared `sql.DB` | `DATABASE_MAX_OPEN_CONNS` | `25` | sqlc `Queries` and GORM — both clients are backed by the **same** `*sql.DB` created in `main.go` |+| Shared `pgxpool` | `DATABASE_VECTOR_MAX_CONNS` | `10` | All `pgvector.Store` instances: every flow/assistant tool executor + knowledge API |-Additional knob: `DB_MAX_IDLE_CONNS` (default `5`) — idle connections kept open between requests.+Additional knob: `DATABASE_MAX_IDLE_CONNS` (default `5`) — idle connections kept open between requests.`NewGorm` accepts the already-configured `*sql.DB` so GORM never opens its own pool:@@ -1234,13 +1234,13 @@ instead of opening individual `pgx.Connect` calls per executor.```Available for client connections = 97- pentagi sql.DB (DB_MAX_OPEN_CONNS) = 25- pentagi pgxpool (DB_VECTOR_MAX_CONNS) = 10- pgexporter = 3- autovacuum workers = 3+ pentagi sql.DB (DATABASE_MAX_OPEN_CONNS) = 25+ pentagi pgxpool (DATABASE_VECTOR_MAX_CONNS) = 10+ pgexporter = 3+ autovacuum workers = 3─────────────────────────────────────────────- Total consumed = 41- Free buffer = 56 (≈ 58%)+ Total consumed = 41+ Free buffer = 56 (≈ 58%)```Defaults are sized for 10 parallel flows with concurrent API requests. To inspect the livediff --git a/backend/pkg/config/config.go b/backend/pkg/config/config.goindex 3f52212e..9ad47c97 100644--- a/backend/pkg/config/config.go+++ b/backend/pkg/config/config.go@@ -72,6 +72,7 @@ type Config struct {EmbeddingStripNewLines bool `env:"EMBEDDING_STRIP_NEW_LINES" envDefault:"true"`EmbeddingBatchSize int `env:"EMBEDDING_BATCH_SIZE" envDefault:"512"`EmbeddingProvider string `env:"EMBEDDING_PROVIDER" envDefault:"openai"`+ EmbeddingMaxTextBytes int `env:"EMBEDDING_MAX_TEXT_BYTES" envDefault:"8192"`// === Chain Summarization Engine ===SummarizerPreserveLast bool `env:"SUMMARIZER_PRESERVE_LAST" envDefault:"true"`@@ -226,13 +227,9 @@ type Config struct {AgentPlanningStepEnabled bool `env:"AGENT_PLANNING_STEP_ENABLED" envDefault:"false"`// === Database Connection Pool Sizing ===- // See backend/docs/database.md §Connection Pooling for budget calculation and- // operational commands. Both sqlc (Queries) and GORM share a single *sql.DB,- // so DB_MAX_OPEN_CONNS is the total sql.DB budget for the process.- // Ensure DB_MAX_OPEN_CONNS + DB_VECTOR_MAX_CONNS < Postgres max_connections.- DBMaxOpenConns int `env:"DB_MAX_OPEN_CONNS" envDefault:"25"`- DBMaxIdleConns int `env:"DB_MAX_IDLE_CONNS" envDefault:"5"`- DBVectorMaxConns int `env:"DB_VECTOR_MAX_CONNS" envDefault:"10"`+ DBMaxOpenConns int `env:"DATABASE_MAX_OPEN_CONNS" envDefault:"25"`+ DBMaxIdleConns int `env:"DATABASE_MAX_IDLE_CONNS" envDefault:"5"`+ DBVectorMaxConns int `env:"DATABASE_VECTOR_MAX_CONNS" envDefault:"10"`// PgxPool is the shared pgxpool.Pool for all pgvector stores. Populated by// main after pool creation; NOT sourced from environment variables.diff --git a/backend/pkg/config/config_test.go b/backend/pkg/config/config_test.goindex 68926522..85aa69f6 100644--- a/backend/pkg/config/config_test.go+++ b/backend/pkg/config/config_test.go@@ -278,7 +278,7 @@ func clearConfigEnv(t *testing.T) {"OPEN_AI_KEY", "OPEN_AI_SERVER_URL","ANTHROPIC_API_KEY", "ANTHROPIC_SERVER_URL","EMBEDDING_URL", "EMBEDDING_KEY", "EMBEDDING_MODEL",- "EMBEDDING_STRIP_NEW_LINES", "EMBEDDING_BATCH_SIZE", "EMBEDDING_PROVIDER",+ "EMBEDDING_STRIP_NEW_LINES", "EMBEDDING_BATCH_SIZE", "EMBEDDING_MAX_TEXT_BYTES", "EMBEDDING_PROVIDER","SUMMARIZER_PRESERVE_LAST", "SUMMARIZER_USE_QA", "SUMMARIZER_SUM_MSG_HUMAN_IN_QA","SUMMARIZER_LAST_SEC_BYTES", "SUMMARIZER_MAX_BP_BYTES","SUMMARIZER_MAX_QA_SECTIONS", "SUMMARIZER_MAX_QA_BYTES", "SUMMARIZER_KEEP_QA_SECTIONS",diff --git a/backend/pkg/database/knowledge.sql.go b/backend/pkg/database/knowledge.sql.goindex b6c6c8db..c11337d3 100644--- a/backend/pkg/database/knowledge.sql.go+++ b/backend/pkg/database/knowledge.sql.go@@ -8,7 +8,9 @@ package databaseimport ("context""database/sql"+ "encoding/json"+ "github.com/google/uuid""github.com/sqlc-dev/pqtype")@@ -35,8 +37,8 @@ WHERE uuid::text = $1`// Delete a knowledge document by UUID (admin — no user_id check).-func (q *Queries) DeleteKnowledgeDocument(ctx context.Context, uuid sql.NullString) error {- _, err := q.db.ExecContext(ctx, deleteKnowledgeDocument, uuid)+func (q *Queries) DeleteKnowledgeDocument(ctx context.Context, argUuid sql.NullString) error {+ _, err := q.db.ExecContext(ctx, deleteKnowledgeDocument, argUuid)return err}@@ -76,8 +78,8 @@ type GetKnowledgeDocumentRow struct {}// Fetch a single knowledge document by its UUID (admin view — no user_id check).-func (q *Queries) GetKnowledgeDocument(ctx context.Context, uuid string) (GetKnowledgeDocumentRow, error) {- row := q.db.QueryRowContext(ctx, getKnowledgeDocument, uuid)+func (q *Queries) GetKnowledgeDocument(ctx context.Context, argUuid string) (GetKnowledgeDocumentRow, error) {+ row := q.db.QueryRowContext(ctx, getKnowledgeDocument, argUuid)var i GetKnowledgeDocumentRowerr := row.Scan(&i.ID, &i.Document, &i.Cmetadata)return i, err@@ -114,6 +116,41 @@ func (q *Queries) GetUserKnowledgeDocument(ctx context.Context, arg GetUserKnowlreturn i, err}+const insertKnowledgeDocument = `-- name: InsertKnowledgeDocument :one+INSERT INTO langchain_pg_embedding (uuid, document, embedding, cmetadata, collection_id)+SELECT+ $1::uuid,+ $2,+ $3::vector,+ $4::json,+ c.uuid+FROM langchain_pg_collection c+WHERE c.name = 'langchain'+RETURNING uuid::text AS id+`++type InsertKnowledgeDocumentParams struct {+ Uuid uuid.UUID `json:"uuid"`+ Document sql.NullString `json:"document"`+ Embedding interface{} `json:"embedding"`+ Cmetadata json.RawMessage `json:"cmetadata"`+}++// Insert a document with a pre-computed embedding vector and return its UUID.+// embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'+// cmetadata must be valid JSON text.+func (q *Queries) InsertKnowledgeDocument(ctx context.Context, arg InsertKnowledgeDocumentParams) (string, error) {+ row := q.db.QueryRowContext(ctx, insertKnowledgeDocument,+ arg.Uuid,+ arg.Document,+ arg.Embedding,+ arg.Cmetadata,+ )+ var id string+ err := row.Scan(&id)+ return id, err+}+const listAllKnowledgeDocuments = `-- name: ListAllKnowledgeDocuments :manySELECTe.uuid::text AS id,diff --git a/backend/pkg/database/knowledge/knowledge.go b/backend/pkg/database/knowledge/knowledge.goindex 794210a8..b8e13845 100644--- a/backend/pkg/database/knowledge/knowledge.go+++ b/backend/pkg/database/knowledge/knowledge.go@@ -14,8 +14,8 @@ import ("pentagi/pkg/graph/subscriptions""pentagi/pkg/providers/embeddings"+ "github.com/google/uuid""github.com/sqlc-dev/pqtype"- "github.com/vxcontrol/langchaingo/schema""github.com/vxcontrol/langchaingo/vectorstores")@@ -54,10 +54,11 @@ type KnowledgeStore interface {}type knowledgeStore struct {- db database.Querier- store vectorstores.VectorStore // may be nil when no embedder is configured- embedder embeddings.Embedder // used for computing new embeddings on create/update- newKnp PublisherFactory+ db database.Querier+ store vectorstores.VectorStore // may be nil when no embedder is configured+ embedder embeddings.Embedder // used for computing new embeddings on create/update+ newKnp PublisherFactory+ maxEmbeddingBytes int}// NewKnowledgeStore constructs a KnowledgeStore.@@ -66,17 +67,25 @@ type knowledgeStore struct {// list/get/delete still work.// - newKnp is called with the acting user's ID on each write to create a// correctly scoped event publisher.+// - maxEmbeddingBytes is the maximum byte size of text sent to the embedding+// model. Text is truncated to this limit before embedding to avoid token+// limit errors; the full original text is always stored in the database.func NewKnowledgeStore(db database.Querier,store vectorstores.VectorStore,embedder embeddings.Embedder,newKnp PublisherFactory,+ maxEmbeddingBytes int,) KnowledgeStore {+ if maxEmbeddingBytes <= 0 {+ maxEmbeddingBytes = 8192+ }return &knowledgeStore{- db: db,- store: store,- embedder: embedder,- newKnp: newKnp,+ db: db,+ store: store,+ embedder: embedder,+ newKnp: newKnp,+ maxEmbeddingBytes: maxEmbeddingBytes,}}@@ -347,6 +356,12 @@ func (ks *knowledgeStore) doSearch(ctx context.Context, userID int64, query strilimit = defaultSearchLimit}+ // Truncate query to embedding size limit to avoid token limit errors.+ // The search quality is preserved since the most relevant context is at the start.+ if len(query) > ks.maxEmbeddingBytes {+ query = query[:ks.maxEmbeddingBytes]+ }+// Compute query embedding.vecs, err := ks.embedder.EmbedDocuments(ctx, []string{query})if err != nil {@@ -443,7 +458,7 @@ func passesSearchFilter(doc *model.KnowledgeDocument, filter *model.KnowledgeFil// ---- CreateDocument ---------------------------------------------------------func (ks *knowledgeStore) CreateDocument(ctx context.Context, userID int64, input model.CreateKnowledgeDocumentInput) (*model.KnowledgeDocument, error) {- if err := ks.requireStore(); err != nil {+ if err := ks.requireEmbedder(); err != nil {return nil, err}@@ -470,19 +485,36 @@ func (ks *knowledgeStore) CreateDocument(ctx context.Context, userID int64, inpumeta.PartSize = len(content)meta.TotalSize = len(content)- metaMap := metaToMap(meta)- ids, err := ks.store.AddDocuments(ctx, []schema.Document{{- PageContent: content,- Metadata: metaMap,- }})+ // Truncate to embedding size limit for vector computation; full content goes to DB.+ embeddingText := content+ if len(embeddingText) > ks.maxEmbeddingBytes {+ embeddingText = embeddingText[:ks.maxEmbeddingBytes]+ }+ vecs, err := ks.embedder.EmbedDocuments(ctx, []string{embeddingText})if err != nil {- return nil, fmt.Errorf("knowledge: create document: %w", err)+ return nil, fmt.Errorf("knowledge: compute embedding: %w", err)+ }+ if len(vecs) == 0 {+ return nil, fmt.Errorf("knowledge: embedder returned no vectors")+ }++ cmJSON, err := metaToJSON(meta)+ if err != nil {+ return nil, fmt.Errorf("knowledge: marshal cmetadata: %w", err)}- if len(ids) == 0 {- return nil, fmt.Errorf("knowledge: create returned no IDs")++ id := uuid.New()+ docID, err := ks.db.InsertKnowledgeDocument(ctx, database.InsertKnowledgeDocumentParams{+ Uuid: id,+ Document: nsOf(content),+ Embedding: formatVector(vecs[0]),+ Cmetadata: cmJSON.RawMessage,+ })+ if err != nil {+ return nil, fmt.Errorf("knowledge: create document: %w", err)}- doc := metaToModelDoc(ids[0], content, meta)+ doc := metaToModelDoc(docID, content, meta)ks.newKnp(userID).KnowledgeDocumentCreated(ctx, doc)return doc, nil}@@ -591,8 +623,13 @@ func (ks *knowledgeStore) doUpdate(ctx context.Context, userID int64, id string,meta.TotalSize = existing.TotalSize + deltaContentLen}- // Compute new embedding.- vecs, err := ks.embedder.EmbedDocuments(ctx, []string{content})+ // Compute new embedding. Truncate to maxEmbeddingBytes to avoid token limit+ // errors; the full content is stored in the document column.+ embeddingText := content+ if len(embeddingText) > ks.maxEmbeddingBytes {+ embeddingText = embeddingText[:ks.maxEmbeddingBytes]+ }+ vecs, err := ks.embedder.EmbedDocuments(ctx, []string{embeddingText})if err != nil {return nil, fmt.Errorf("knowledge: compute embedding: %w", err)}diff --git a/backend/pkg/database/querier.go b/backend/pkg/database/querier.goindex 5517efbb..8d80eb45 100644--- a/backend/pkg/database/querier.go+++ b/backend/pkg/database/querier.go@@ -237,6 +237,10 @@ type Querier interface {GetUserTotalToolcallsStats(ctx context.Context, userID int64) (GetUserTotalToolcallsStatsRow, error)GetUserTotalUsageStats(ctx context.Context, userID int64) (GetUserTotalUsageStatsRow, error)GetUsers(ctx context.Context) ([]GetUsersRow, error)+ // Insert a document with a pre-computed embedding vector and return its UUID.+ // embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'+ // cmetadata must be valid JSON text.+ InsertKnowledgeDocument(ctx context.Context, arg InsertKnowledgeDocumentParams) (string, error)// List all knowledge documents excluding the noisy memory type (admin view).ListAllKnowledgeDocuments(ctx context.Context) ([]ListAllKnowledgeDocumentsRow, error)// List non-memory knowledge documents belonging to a specific flow (admin scoped).diff --git a/backend/pkg/server/router.go b/backend/pkg/server/router.goindex 628f2c7c..5404e35d 100644--- a/backend/pkg/server/router.go+++ b/backend/pkg/server/router.go@@ -148,7 +148,7 @@ func NewRouter(}}var knowledgeStore knowledge.KnowledgeStore- knowledgeStore = knowledge.NewKnowledgeStore(db, pgStore, embedder, subscriptions.NewKnowledgePublisher)+ knowledgeStore = knowledge.NewKnowledgeStore(db, pgStore, embedder, subscriptions.NewKnowledgePublisher, cfg.EmbeddingMaxTextBytes)// ---- Anonymizer replacer ------------------------------------------------// Shared singleton used by the GraphQL anonymizeText mutation.diff --git a/backend/pkg/tools/code.go b/backend/pkg/tools/code.goindex c8eb06ee..238860f2 100644--- a/backend/pkg/tools/code.go+++ b/backend/pkg/tools/code.go@@ -4,12 +4,14 @@ import ("context""encoding/json""fmt"+ "maps""strings""pentagi/pkg/database""pentagi/pkg/graph/model"obs "pentagi/pkg/observability""pentagi/pkg/observability/langfuse"+ "pentagi/pkg/providers/embeddings""github.com/sirupsen/logrus""github.com/vxcontrol/cloud/anonymizer"@@ -27,14 +29,17 @@ const ()type code struct {- userID int64- flowID int64- taskID *int64- subtaskID *int64- replacer anonymizer.Replacer- store *pgvector.Store- vslp VectorStoreLogProvider- knp KnowledgeProvider+ userID int64+ flowID int64+ taskID *int64+ subtaskID *int64+ replacer anonymizer.Replacer+ store *pgvector.Store+ embedder embeddings.Embedder+ db database.Querier+ maxEmbeddingBytes int+ vslp VectorStoreLogProvider+ knp KnowledgeProvider}func NewCodeTool(@@ -43,18 +48,24 @@ func NewCodeTool(taskID, subtaskID *int64,replacer anonymizer.Replacer,store *pgvector.Store,+ embedder embeddings.Embedder,+ db database.Querier,+ maxEmbeddingBytes int,vslp VectorStoreLogProvider,knp KnowledgeProvider,) Tool {return &code{- userID: userID,- flowID: flowID,- taskID: taskID,- subtaskID: subtaskID,- replacer: replacer,- store: store,- vslp: vslp,- knp: knp,+ userID: userID,+ flowID: flowID,+ taskID: taskID,+ subtaskID: subtaskID,+ replacer: replacer,+ store: store,+ embedder: embedder,+ db: db,+ maxEmbeddingBytes: maxEmbeddingBytes,+ vslp: vslp,+ knp: knp,}}@@ -215,22 +226,28 @@ func (c *code) Handle(ctx context.Context, name string, args json.RawMessage) (sreturn "", fmt.Errorf("failed to unmarshal %s store code action arguments: %w", name, err)}- buffer := strings.Builder{}- buffer.WriteString(action.Explanation)- buffer.WriteString(fmt.Sprintf("\n\n```%s\n\n", action.Lang))- buffer.WriteString(action.Code)- buffer.WriteString("\n```")+ renderedCode := c.renderCode(action.Explanation, action.Lang, action.Code)++ // Anonymize before anything else so all downstream paths (including error+ // branches that emit langfuse events) only ever expose the anonymized form.+ var (+ anonymizedCode = c.replacer.ReplaceString(renderedCode)+ anonymizedQuestion = c.replacer.ReplaceString(action.Question)+ anonymizedDescription = c.replacer.ReplaceString(action.Description)+ )+ eventMetadata := map[string]any{+ "tool_name": name,+ "code_lang": action.Lang,+ "message": action.Message,+ "description": anonymizedDescription,+ "doc_type": codeVectorStoreDefaultType,+ }opts := []langfuse.EventOption{langfuse.WithEventName("store code samples to vector store"),langfuse.WithEventInput(action.Question),- langfuse.WithEventOutput(buffer.String()),- langfuse.WithEventMetadata(map[string]any{- "tool_name": name,- "code_lang": action.Lang,- "message": action.Message,- "doc_type": codeVectorStoreDefaultType,- }),+ langfuse.WithEventOutput(anonymizedCode),+ langfuse.WithEventMetadata(eventMetadata),}logger = logger.WithFields(logrus.Fields{@@ -239,49 +256,81 @@ func (c *code) Handle(ctx context.Context, name string, args json.RawMessage) (s"code": action.Code[:min(len(action.Code), 1000)],})+ // Build common metadata for the document.+ metadata := map[string]any{+ "user_id": c.userID,+ "flow_id": c.flowID,+ "doc_type": codeVectorStoreDefaultType,+ "code_lang": action.Lang,+ "question": anonymizedQuestion,+ "description": anonymizedDescription,+ "part_size": len(anonymizedCode),+ "total_size": len(anonymizedCode),+ }+ if c.taskID != nil {+ metadata["task_id"] = *c.taskID+ }+ if c.subtaskID != nil {+ metadata["subtask_id"] = *c.subtaskID+ }+var (- anonymizedCode = c.replacer.ReplaceString(buffer.String())- anonymizedQuestion = c.replacer.ReplaceString(action.Question)+ docs []schema.Document+ err error+ ids []string)- docs, err := documentloaders.NewText(strings.NewReader(anonymizedCode)).Load(ctx)- if err != nil {- observation.Event(append(opts,- langfuse.WithEventStatus(err.Error()),- langfuse.WithEventLevel(langfuse.ObservationLevelError),- )...)- logger.WithError(err).Error("failed to load document")- return "", fmt.Errorf("failed to load document: %w", err)- }-- for _, doc := range docs {- if doc.Metadata == nil {- doc.Metadata = map[string]any{}+ if len(anonymizedCode) <= c.maxEmbeddingBytes || c.embedder == nil {+ // Fast path: document fits within the embedding limit — use AddDocuments normally.+ docs, err = documentloaders.NewText(strings.NewReader(anonymizedCode)).Load(ctx)+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to load document")+ return "", fmt.Errorf("failed to load document: %w", err)}- doc.Metadata["user_id"] = c.userID- doc.Metadata["flow_id"] = c.flowID- if c.taskID != nil {- doc.Metadata["task_id"] = *c.taskID+ for i := range docs {+ if docs[i].Metadata == nil {+ docs[i].Metadata = map[string]any{}+ }+ maps.Copy(docs[i].Metadata, metadata)+ docs[i].Metadata["part_size"] = len(docs[i].PageContent)}- if c.subtaskID != nil {- doc.Metadata["subtask_id"] = *c.subtaskID+ ids, err = c.store.AddDocuments(ctx, docs)+ eventMetadata["ids"] = ids+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to store code sample")+ return "", fmt.Errorf("failed to store code sample: %w", err)}- doc.Metadata["doc_type"] = codeVectorStoreDefaultType- doc.Metadata["code_lang"] = action.Lang- doc.Metadata["question"] = anonymizedQuestion- doc.Metadata["description"] = action.Description- doc.Metadata["part_size"] = len(doc.PageContent)- doc.Metadata["total_size"] = len(anonymizedCode)- }+ } else {+ // Slow path: document exceeds embedding limit — embed truncated text,+ // store the full anonymized document in the database.+ embeddingText := truncateForEmbedding(anonymizedCode, c.maxEmbeddingBytes)- ids, err := c.store.AddDocuments(ctx, docs)- if err != nil {- observation.Event(append(opts,- langfuse.WithEventStatus(err.Error()),- langfuse.WithEventLevel(langfuse.ObservationLevelError),- )...)- logger.WithError(err).Error("failed to store code sample")- return "", fmt.Errorf("failed to store code sample: %w", err)+ id, err := storeDocumentWithEmbeddingLimit(ctx, c.db, c.embedder,+ embeddingText, anonymizedCode, metadata)+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to store code sample with embedding limit")+ return "", fmt.Errorf("failed to store code sample: %w", err)+ }+ ids = []string{id}+ docs = []schema.Document{+ {+ PageContent: anonymizedCode,+ Metadata: metadata,+ },+ }+ eventMetadata["ids"] = ids}observation.Event(append(opts,@@ -292,20 +341,16 @@ func (c *code) Handle(ctx context.Context, name string, args json.RawMessage) (sif c.knp != nil {codeLang := action.Lang- for i, doc := range docs {- if i >= len(ids) {- break- }- desc := action.Description+ for _, id := range ids {knDoc := &model.KnowledgeDocument{- ID: ids[i],+ ID: id,UserID: c.userID,DocType: model.KnowledgeDocTypeCode,- Content: doc.PageContent,+ Content: anonymizedCode,Question: anonymizedQuestion,- Description: &desc,+ Description: &anonymizedDescription,CodeLang: &codeLang,- PartSize: len(doc.PageContent),+ PartSize: len(anonymizedCode),TotalSize: len(anonymizedCode),Manual: false,}@@ -341,7 +386,7 @@ func (c *code) Handle(ctx context.Context, name string, args json.RawMessage) (sfiltersData,action.Question,database.VecstoreActionTypeStore,- buffer.String(),+ renderedCode,c.taskID,c.subtaskID,)@@ -358,3 +403,15 @@ func (c *code) Handle(ctx context.Context, name string, args json.RawMessage) (sfunc (c *code) IsAvailable() bool {return c.store != nil}++func (c *code) renderCode(explanation, lang, code string) string {+ buffer := strings.Builder{}+ buffer.WriteString(explanation)+ buffer.WriteString("\n\n")+ buffer.WriteString("```")+ buffer.WriteString(lang)+ buffer.WriteString("\n")+ buffer.WriteString(code)+ buffer.WriteString("\n```")+ return buffer.String()+}diff --git a/backend/pkg/tools/guide.go b/backend/pkg/tools/guide.goindex 07e77030..a0d304c5 100644--- a/backend/pkg/tools/guide.go+++ b/backend/pkg/tools/guide.go@@ -4,12 +4,14 @@ import ("context""encoding/json""fmt"+ "maps""strings""pentagi/pkg/database""pentagi/pkg/graph/model"obs "pentagi/pkg/observability""pentagi/pkg/observability/langfuse"+ "pentagi/pkg/providers/embeddings""github.com/sirupsen/logrus""github.com/vxcontrol/cloud/anonymizer"@@ -27,32 +29,41 @@ const ()type guide struct {- userID int64- flowID int64- taskID *int64- subtaskID *int64- replacer anonymizer.Replacer- store *pgvector.Store- vslp VectorStoreLogProvider- knp KnowledgeProvider+ userID int64+ flowID int64+ taskID *int64+ subtaskID *int64+ replacer anonymizer.Replacer+ store *pgvector.Store+ embedder embeddings.Embedder+ db database.Querier+ maxEmbeddingBytes int+ vslp VectorStoreLogProvider+ knp KnowledgeProvider}func NewGuideTool(userID int64, flowID int64, taskID, subtaskID *int64,replacer anonymizer.Replacer,store *pgvector.Store,+ embedder embeddings.Embedder,+ db database.Querier,+ maxEmbeddingBytes int,vslp VectorStoreLogProvider,knp KnowledgeProvider,) Tool {return &guide{- userID: userID,- flowID: flowID,- taskID: taskID,- subtaskID: subtaskID,- replacer: replacer,- store: store,- vslp: vslp,- knp: knp,+ userID: userID,+ flowID: flowID,+ taskID: taskID,+ subtaskID: subtaskID,+ replacer: replacer,+ store: store,+ embedder: embedder,+ db: db,+ maxEmbeddingBytes: maxEmbeddingBytes,+ vslp: vslp,+ knp: knp,}}@@ -214,16 +225,25 @@ func (g *guide) Handle(ctx context.Context, name string, args json.RawMessage) (guide := fmt.Sprintf("Question:\n%s\n\nGuide:\n%s", action.Question, action.Guide)+ // Anonymize before anything else so all downstream paths (including error+ // branches that emit langfuse events) only ever expose the anonymized form.+ var (+ anonymizedGuide = g.replacer.ReplaceString(guide)+ anonymizedQuestion = g.replacer.ReplaceString(action.Question)+ anonymizedGuideOnly = g.replacer.ReplaceString(action.Guide) // used in slow-path embedding text+ )++ eventMetadata := map[string]any{+ "tool_name": name,+ "message": action.Message,+ "doc_type": guideVectorStoreDefaultType,+ "guide_type": action.Type,+ }opts := []langfuse.EventOption{langfuse.WithEventName("store guide to vector store"),langfuse.WithEventInput(action.Question),- langfuse.WithEventOutput(guide),- langfuse.WithEventMetadata(map[string]any{- "tool_name": name,- "message": action.Message,- "doc_type": guideVectorStoreDefaultType,- "guide_type": action.Type,- }),+ langfuse.WithEventOutput(anonymizedGuide),+ langfuse.WithEventMetadata(eventMetadata),}logger = logger.WithFields(logrus.Fields{@@ -232,48 +252,82 @@ func (g *guide) Handle(ctx context.Context, name string, args json.RawMessage) ("guide": action.Guide[:min(len(action.Guide), 1000)],})+ // Build common metadata for the document.+ metadata := map[string]any{+ "user_id": g.userID,+ "flow_id": g.flowID,+ "doc_type": guideVectorStoreDefaultType,+ "guide_type": action.Type,+ "question": anonymizedQuestion,+ "part_size": len(anonymizedGuide),+ "total_size": len(anonymizedGuide),+ }+ if g.taskID != nil {+ metadata["task_id"] = *g.taskID+ }+ if g.subtaskID != nil {+ metadata["subtask_id"] = *g.subtaskID+ }+var (- anonymizedGuide = g.replacer.ReplaceString(guide)- anonymizedQuestion = g.replacer.ReplaceString(action.Question)+ docs []schema.Document+ ids []string+ err error)- docs, err := documentloaders.NewText(strings.NewReader(anonymizedGuide)).Load(ctx)- if err != nil {- observation.Event(append(opts,- langfuse.WithEventStatus(err.Error()),- langfuse.WithEventLevel(langfuse.ObservationLevelError),- )...)- logger.WithError(err).Error("failed to load document")- return "", fmt.Errorf("failed to load document: %w", err)- }-- for _, doc := range docs {- if doc.Metadata == nil {- doc.Metadata = map[string]any{}+ if len(anonymizedGuide) <= g.maxEmbeddingBytes || g.embedder == nil {+ // Fast path: document fits within the embedding limit.+ docs, err = documentloaders.NewText(strings.NewReader(anonymizedGuide)).Load(ctx)+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to load document")+ return "", fmt.Errorf("failed to load document: %w", err)}- doc.Metadata["user_id"] = g.userID- doc.Metadata["flow_id"] = g.flowID- if g.taskID != nil {- doc.Metadata["task_id"] = *g.taskID+ for i := range docs {+ if docs[i].Metadata == nil {+ docs[i].Metadata = map[string]any{}+ }+ maps.Copy(docs[i].Metadata, metadata)+ docs[i].Metadata["part_size"] = len(docs[i].PageContent)}- if g.subtaskID != nil {- doc.Metadata["subtask_id"] = *g.subtaskID+ ids, err = g.store.AddDocuments(ctx, docs)+ eventMetadata["ids"] = ids+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to store guide")+ return "", fmt.Errorf("failed to store guide: %w", err)}- doc.Metadata["doc_type"] = guideVectorStoreDefaultType- doc.Metadata["guide_type"] = action.Type- doc.Metadata["question"] = anonymizedQuestion- doc.Metadata["part_size"] = len(doc.PageContent)- doc.Metadata["total_size"] = len(anonymizedGuide)- }-- ids, err := g.store.AddDocuments(ctx, docs)- if err != nil {- observation.Event(append(opts,- langfuse.WithEventStatus(err.Error()),- langfuse.WithEventLevel(langfuse.ObservationLevelError),- )...)- logger.WithError(err).Error("failed to store guide")- return "", fmt.Errorf("failed to store guide: %w", err)+ } else {+ // Slow path: Guide field exceeds embedding limit.+ // Template: "Question:\n{question}\n\nGuide:\n{guide}"+ prefix := "Question:\n" + anonymizedQuestion + "\n\nGuide:\n"+ available := max(g.maxEmbeddingBytes-len(prefix), 0)+ embeddingText := prefix + truncateForEmbedding(anonymizedGuideOnly, available)++ id, err := storeDocumentWithEmbeddingLimit(ctx, g.db, g.embedder,+ embeddingText, anonymizedGuide, metadata)+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to store guide with embedding limit")+ return "", fmt.Errorf("failed to store guide: %w", err)+ }+ ids = []string{id}+ docs = []schema.Document{+ {+ PageContent: anonymizedGuide,+ Metadata: metadata,+ },+ }+ eventMetadata["ids"] = ids}observation.Event(append(opts,@@ -284,18 +338,15 @@ func (g *guide) Handle(ctx context.Context, name string, args json.RawMessage) (if g.knp != nil {guideType := model.KnowledgeGuideType(action.Type)- for i, doc := range docs {- if i >= len(ids) {- break- }+ for _, id := range ids {knDoc := &model.KnowledgeDocument{- ID: ids[i],+ ID: id,UserID: g.userID,DocType: model.KnowledgeDocTypeGuide,- Content: doc.PageContent,+ Content: anonymizedGuide,Question: anonymizedQuestion,GuideType: &guideType,- PartSize: len(doc.PageContent),+ PartSize: len(anonymizedGuide),TotalSize: len(anonymizedGuide),Manual: false,}diff --git a/backend/pkg/tools/search.go b/backend/pkg/tools/search.goindex e1c9deca..ab852d89 100644--- a/backend/pkg/tools/search.go+++ b/backend/pkg/tools/search.go@@ -4,12 +4,14 @@ import ("context""encoding/json""fmt"+ "maps""strings""pentagi/pkg/database""pentagi/pkg/graph/model"obs "pentagi/pkg/observability""pentagi/pkg/observability/langfuse"+ "pentagi/pkg/providers/embeddings""github.com/sirupsen/logrus""github.com/vxcontrol/cloud/anonymizer"@@ -27,14 +29,17 @@ const ()type search struct {- userID int64- flowID int64- taskID *int64- subtaskID *int64- replacer anonymizer.Replacer- store *pgvector.Store- vslp VectorStoreLogProvider- knp KnowledgeProvider+ userID int64+ flowID int64+ taskID *int64+ subtaskID *int64+ replacer anonymizer.Replacer+ store *pgvector.Store+ embedder embeddings.Embedder+ db database.Querier+ maxEmbeddingBytes int+ vslp VectorStoreLogProvider+ knp KnowledgeProvider}func NewSearchTool(@@ -43,18 +48,24 @@ func NewSearchTool(taskID, subtaskID *int64,replacer anonymizer.Replacer,store *pgvector.Store,+ embedder embeddings.Embedder,+ db database.Querier,+ maxEmbeddingBytes int,vslp VectorStoreLogProvider,knp KnowledgeProvider,) Tool {return &search{- userID: userID,- flowID: flowID,- taskID: taskID,- subtaskID: subtaskID,- replacer: replacer,- store: store,- vslp: vslp,- knp: knp,+ userID: userID,+ flowID: flowID,+ taskID: taskID,+ subtaskID: subtaskID,+ replacer: replacer,+ store: store,+ embedder: embedder,+ db: db,+ maxEmbeddingBytes: maxEmbeddingBytes,+ vslp: vslp,+ knp: knp,}}@@ -210,16 +221,24 @@ func (s *search) Handle(ctx context.Context, name string, args json.RawMessage)return "", fmt.Errorf("failed to unmarshal %s store answer action arguments: %w", name, err)}+ // Anonymize before anything else so all downstream paths (including error+ // branches that emit langfuse events) only ever expose the anonymized form.+ var (+ anonymizedAnswer = s.replacer.ReplaceString(action.Answer)+ anonymizedQuestion = s.replacer.ReplaceString(action.Question)+ )++ eventMetadata := map[string]any{+ "tool_name": name,+ "message": action.Message,+ "doc_type": searchVectorStoreDefaultType,+ "answer_type": action.Type,+ }opts := []langfuse.EventOption{langfuse.WithEventName("store search answer to vector store"),langfuse.WithEventInput(action.Question),- langfuse.WithEventOutput(action.Answer),- langfuse.WithEventMetadata(map[string]any{- "tool_name": name,- "message": action.Message,- "doc_type": searchVectorStoreDefaultType,- "answer_type": action.Type,- }),+ langfuse.WithEventOutput(anonymizedAnswer),+ langfuse.WithEventMetadata(eventMetadata),}logger = logger.WithFields(logrus.Fields{@@ -228,44 +247,80 @@ func (s *search) Handle(ctx context.Context, name string, args json.RawMessage)"answer": action.Answer[:min(len(action.Answer), 1000)],})+ // Build common metadata for the document.+ metadata := map[string]any{+ "user_id": s.userID,+ "flow_id": s.flowID,+ "doc_type": searchVectorStoreDefaultType,+ "answer_type": action.Type,+ "question": anonymizedQuestion,+ "part_size": len(anonymizedAnswer),+ "total_size": len(anonymizedAnswer),+ }+ if s.taskID != nil {+ metadata["task_id"] = *s.taskID+ }+ if s.subtaskID != nil {+ metadata["subtask_id"] = *s.subtaskID+ }+var (- anonymizedAnswer = s.replacer.ReplaceString(action.Answer)- anonymizedQuestion = s.replacer.ReplaceString(action.Question)+ docs []schema.Document+ ids []string+ err error)- docs, err := documentloaders.NewText(strings.NewReader(anonymizedAnswer)).Load(ctx)- if err != nil {- observation.Event(append(opts,- langfuse.WithEventStatus(err.Error()),- langfuse.WithEventLevel(langfuse.ObservationLevelError),- )...)- logger.WithError(err).Error("failed to load document")- return "", fmt.Errorf("failed to load document: %w", err)- }-- for _, doc := range docs {- if doc.Metadata == nil {- doc.Metadata = map[string]any{}+ if len(anonymizedAnswer) <= s.maxEmbeddingBytes || s.embedder == nil {+ // Fast path: answer fits within the embedding limit.+ docs, err = documentloaders.NewText(strings.NewReader(anonymizedAnswer)).Load(ctx)+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to load document")+ return "", fmt.Errorf("failed to load document: %w", err)}- doc.Metadata["user_id"] = s.userID- doc.Metadata["flow_id"] = s.flowID- doc.Metadata["task_id"] = s.taskID- doc.Metadata["subtask_id"] = s.subtaskID- doc.Metadata["doc_type"] = searchVectorStoreDefaultType- doc.Metadata["answer_type"] = action.Type- doc.Metadata["question"] = anonymizedQuestion- doc.Metadata["part_size"] = len(doc.PageContent)- doc.Metadata["total_size"] = len(anonymizedAnswer)- }+ for i := range docs {+ if docs[i].Metadata == nil {+ docs[i].Metadata = map[string]any{}+ }+ maps.Copy(docs[i].Metadata, metadata)+ docs[i].Metadata["part_size"] = len(docs[i].PageContent)+ }+ ids, err = s.store.AddDocuments(ctx, docs)+ eventMetadata["ids"] = ids+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to store answer for question")+ return "", fmt.Errorf("failed to store answer for question: %w", err)+ }+ } else {+ // Slow path: Answer field exceeds embedding limit.+ // PageContent is just the answer text, so overhead = 0.+ embeddingText := truncateForEmbedding(anonymizedAnswer, s.maxEmbeddingBytes)- ids, err := s.store.AddDocuments(ctx, docs)- if err != nil {- observation.Event(append(opts,- langfuse.WithEventStatus(err.Error()),- langfuse.WithEventLevel(langfuse.ObservationLevelError),- )...)- logger.WithError(err).Error("failed to store answer for question")- return "", fmt.Errorf("failed to store answer for question: %w", err)+ id, err := storeDocumentWithEmbeddingLimit(ctx, s.db, s.embedder,+ embeddingText, anonymizedAnswer, metadata)+ if err != nil {+ observation.Event(append(opts,+ langfuse.WithEventStatus(err.Error()),+ langfuse.WithEventLevel(langfuse.ObservationLevelError),+ )...)+ logger.WithError(err).Error("failed to store answer with embedding limit")+ return "", fmt.Errorf("failed to store answer for question: %w", err)+ }+ ids = []string{id}+ docs = []schema.Document{+ {+ PageContent: anonymizedAnswer,+ Metadata: metadata,+ },+ }+ eventMetadata["ids"] = ids}observation.Event(append(opts,@@ -276,18 +331,15 @@ func (s *search) Handle(ctx context.Context, name string, args json.RawMessage)if s.knp != nil {answerType := model.KnowledgeAnswerType(action.Type)- for i, doc := range docs {- if i >= len(ids) {- break- }+ for _, id := range ids {knDoc := &model.KnowledgeDocument{- ID: ids[i],+ ID: id,UserID: s.userID,DocType: model.KnowledgeDocTypeAnswer,- Content: doc.PageContent,+ Content: anonymizedAnswer,Question: anonymizedQuestion,AnswerType: &answerType,- PartSize: len(doc.PageContent),+ PartSize: len(anonymizedAnswer),TotalSize: len(anonymizedAnswer),Manual: false,}@@ -301,12 +353,17 @@ func (s *search) Handle(ctx context.Context, name string, args json.RawMessage)}if agentCtx, ok := GetAgentContext(ctx); ok {- filtersData, err := json.Marshal(map[string]any{+ data := map[string]any{"doc_type": searchVectorStoreDefaultType,"answer_type": action.Type,- "task_id": s.taskID,- "subtask_id": s.subtaskID,- })+ }+ if s.taskID != nil {+ data["task_id"] = *s.taskID+ }+ if s.subtaskID != nil {+ data["subtask_id"] = *s.subtaskID+ }+ filtersData, err := json.Marshal(data)if err != nil {logger.WithError(err).Error("failed to marshal filters")return "", fmt.Errorf("failed to marshal filters: %w", err)diff --git a/backend/pkg/tools/tools.go b/backend/pkg/tools/tools.goindex 6e3bc80c..2c72884b 100644--- a/backend/pkg/tools/tools.go+++ b/backend/pkg/tools/tools.go@@ -172,6 +172,7 @@ type flowToolsExecutor struct {db database.Queriercfg *config.Config+ embedder embeddings.Embedderstore *pgvector.StoregraphitiClient *graphiti.Clientimage string@@ -392,6 +393,7 @@ func (fte *flowToolsExecutor) SetImage(image string) {}func (fte *flowToolsExecutor) SetEmbedder(embedder embeddings.Embedder) {+ fte.embedder = embedderif !embedder.IsAvailable() {return}@@ -865,6 +867,9 @@ func (fte *flowToolsExecutor) GetAssistantExecutor(cfg AssistantExecutorConfig)fte.userID, fte.flowID, nil, nil,fte.replacer,fte.store,+ fte.embedder,+ fte.db,+ fte.cfg.EmbeddingMaxTextBytes,fte.vslp,fte.knp,)@@ -877,6 +882,9 @@ func (fte *flowToolsExecutor) GetAssistantExecutor(cfg AssistantExecutorConfig)fte.userID, fte.flowID, nil, nil,fte.replacer,fte.store,+ fte.embedder,+ fte.db,+ fte.cfg.EmbeddingMaxTextBytes,fte.vslp,fte.knp,)@@ -889,6 +897,9 @@ func (fte *flowToolsExecutor) GetAssistantExecutor(cfg AssistantExecutorConfig)fte.userID, fte.flowID, nil, nil,fte.replacer,fte.store,+ fte.embedder,+ fte.db,+ fte.cfg.EmbeddingMaxTextBytes,fte.vslp,fte.knp,)@@ -1174,6 +1185,9 @@ func (fte *flowToolsExecutor) GetInstallerExecutor(cfg InstallerExecutorConfig)cfg.SubtaskID,fte.replacer,fte.store,+ fte.embedder,+ fte.db,+ fte.cfg.EmbeddingMaxTextBytes,fte.vslp,fte.knp,)@@ -1279,6 +1293,9 @@ func (fte *flowToolsExecutor) GetCoderExecutor(cfg CoderExecutorConfig) (Contextcfg.SubtaskID,fte.replacer,fte.store,+ fte.embedder,+ fte.db,+ fte.cfg.EmbeddingMaxTextBytes,fte.vslp,fte.knp,)@@ -1401,6 +1418,9 @@ func (fte *flowToolsExecutor) GetPentesterExecutor(cfg PentesterExecutorConfig)cfg.SubtaskID,fte.replacer,fte.store,+ fte.embedder,+ fte.db,+ fte.cfg.EmbeddingMaxTextBytes,fte.vslp,fte.knp,)@@ -1578,6 +1598,9 @@ func (fte *flowToolsExecutor) GetSearcherExecutor(cfg SearcherExecutorConfig) (Ccfg.SubtaskID,fte.replacer,fte.store,+ fte.embedder,+ fte.db,+ fte.cfg.EmbeddingMaxTextBytes,fte.vslp,fte.knp,)diff --git a/backend/pkg/tools/vecstore_helper.go b/backend/pkg/tools/vecstore_helper.gonew file mode 100644index 00000000..a7c99616--- /dev/null+++ b/backend/pkg/tools/vecstore_helper.go@@ -0,0 +1,78 @@+package tools++import (+ "context"+ "database/sql"+ "encoding/json"+ "fmt"+ "strconv"+ "strings"++ "pentagi/pkg/database"+ "pentagi/pkg/providers/embeddings"++ "github.com/google/uuid"+)++// truncateForEmbedding truncates text to maxBytes bytes for sending to the+// embedding model API. Returns text unchanged if it is already within the limit.+// This avoids token-limit errors (e.g. OpenAI: max 8192 tokens) when a heavy+// field exceeds the model's context window.+func truncateForEmbedding(text string, maxBytes int) string {+ if maxBytes <= 0 || len(text) <= maxBytes {+ return text+ }+ return text[:maxBytes]+}++// storeDocumentWithEmbeddingLimit computes the embedding vector from embeddingText+// (the truncated/prefix version of the content, ≤ maxBytes) and inserts the+// document into the vector store with the full original text as the stored content.+//+// Use this when len(fullText) > maxBytes to avoid embedding API token-limit errors+// while still persisting the complete document for retrieval.+//+// Returns the UUID string of the newly inserted document.+func storeDocumentWithEmbeddingLimit(+ ctx context.Context,+ db database.Querier,+ embedder embeddings.Embedder,+ embeddingText string,+ fullText string,+ metadata map[string]any,+) (string, error) {+ vecs, err := embedder.EmbedDocuments(ctx, []string{embeddingText})+ if err != nil {+ return "", fmt.Errorf("failed to compute embedding: %w", err)+ }+ if len(vecs) == 0 {+ return "", fmt.Errorf("embedder returned no vectors")+ }++ metaJSON, err := json.Marshal(metadata)+ if err != nil {+ return "", fmt.Errorf("failed to marshal metadata: %w", err)+ }++ id := uuid.New()+ docID, err := db.InsertKnowledgeDocument(ctx, database.InsertKnowledgeDocumentParams{+ Uuid: id,+ Document: sql.NullString{String: fullText, Valid: true},+ Embedding: formatVectorFromFloat32s(vecs[0]),+ Cmetadata: json.RawMessage(metaJSON),+ })+ if err != nil {+ return "", fmt.Errorf("failed to insert document with pre-computed embedding: %w", err)+ }++ return docID, nil+}++// formatVectorFromFloat32s converts a float32 slice into a PostgreSQL vector literal '[f1,f2,...]'.+func formatVectorFromFloat32s(v []float32) string {+ strs := make([]string, len(v))+ for i, f := range v {+ strs[i] = strconv.FormatFloat(float64(f), 'f', -1, 32)+ }+ return "[" + strings.Join(strs, ",") + "]"+}diff --git a/backend/sqlc/models/knowledge.sql b/backend/sqlc/models/knowledge.sqlindex a0812409..b06d18bc 100644--- a/backend/sqlc/models/knowledge.sql+++ b/backend/sqlc/models/knowledge.sql@@ -138,3 +138,18 @@ DELETE FROM langchain_pg_embeddingWHERE collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = 'langchain')AND COALESCE(cmetadata ->> 'doc_type', '') = 'memory'AND (cmetadata ->> 'flow_id') = sqlc.arg(flow_id);++-- name: InsertKnowledgeDocument :one+-- Insert a document with a pre-computed embedding vector and return its UUID.+-- embedding must be formatted as a PostgreSQL vector literal: '[f1,f2,...]'+-- cmetadata must be valid JSON text.+INSERT INTO langchain_pg_embedding (uuid, document, embedding, cmetadata, collection_id)+SELECT+ sqlc.arg(uuid)::uuid,+ sqlc.arg(document),+ sqlc.arg(embedding)::vector,+ sqlc.arg(cmetadata)::json,+ c.uuid+FROM langchain_pg_collection c+WHERE c.name = 'langchain'+RETURNING uuid::text AS id;diff --git a/docker-compose.yml b/docker-compose.ymlindex f706b166..20cc37d4 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -86,6 +86,7 @@ services:- EMBEDDING_MODEL=${EMBEDDING_MODEL:-}- EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-}- EMBEDDING_BATCH_SIZE=${EMBEDDING_BATCH_SIZE:-}+ - EMBEDDING_MAX_TEXT_BYTES=${EMBEDDING_MAX_TEXT_BYTES:-}- EMBEDDING_STRIP_NEW_LINES=${EMBEDDING_STRIP_NEW_LINES:-}- SUMMARIZER_PRESERVE_LAST=${SUMMARIZER_PRESERVE_LAST:-}- SUMMARIZER_USE_QA=${SUMMARIZER_USE_QA:-}@@ -131,6 +132,9 @@ services:- OAUTH_GITHUB_CLIENT_ID=${OAUTH_GITHUB_CLIENT_ID:-}- OAUTH_GITHUB_CLIENT_SECRET=${OAUTH_GITHUB_CLIENT_SECRET:-}- DATABASE_URL=postgres://${PENTAGI_POSTGRES_USER:-postgres}:${PENTAGI_POSTGRES_PASSWORD:-postgres}@pgvector:5432/${PENTAGI_POSTGRES_DB:-pentagidb}?sslmode=disable+ - DATABASE_MAX_OPEN_CONNS=${DATABASE_MAX_OPEN_CONNS:-}+ - DATABASE_MAX_IDLE_CONNS=${DATABASE_MAX_IDLE_CONNS:-}+ - DATABASE_VECTOR_MAX_CONNS=${DATABASE_VECTOR_MAX_CONNS:-}- DUCKDUCKGO_ENABLED=${DUCKDUCKGO_ENABLED:-}- DUCKDUCKGO_REGION=${DUCKDUCKGO_REGION:-}- DUCKDUCKGO_SAFESEARCH=${DUCKDUCKGO_SAFESEARCH:-}