Fully autonomous AI Agents system capable of performing complex penetration testing tasks
Go ai-agentsai-security-toolanthropicautonomous-agentsgolanggptgraphqlmulti-agent-systemoffensive-securityopen-sourceopenaipenetration-testingpenetration-testing-toolsreactsecurity-automationsecurity-testingsecurity-toolsself-hosted
feat(database): enhance PostgreSQL connection pooling and configuration
Dmitry Ng077ddce476a6
diff --git a/README.md b/README.mdindex b31b4039..5f7ef98d 100644--- a/README.md+++ b/README.md@@ -2613,6 +2613,61 @@ Optional:- `SERVER_PORT` - Port to run the server (default: `8443`)- `SERVER_USE_SSL` - Enable SSL for the server (default: `false`)+##### PostgreSQL / pgvector connection pool sizing++PentAGI opens two independent connection pools to the same Postgres instance:++| Pool | Env var | Default | Used by |+|---|---|---|---|+| Shared `sql.DB` | `DB_MAX_OPEN_CONNS` | `25` | All sqlc queries and GORM handlers share a single `*sql.DB` |+| Shared `pgxpool` | `DB_VECTOR_MAX_CONNS` | `10` | All pgvector stores (agent memory + knowledge API) share a single pool |++Additional tuning knob:+- `DB_MAX_IDLE_CONNS` — maximum idle connections kept open in the `sql.DB` pool between requests (default: `5`).++**Budget for the stock `vxcontrol/pgvector` image** (`max_connections = 100`, `superuser_reserved_connections = 3`):++```+Available for client connections = 97+ pentagi sql.DB (DB_MAX_OPEN_CONNS) = 25+ pentagi pgxpool (DB_VECTOR_MAX_CONNS) = 10+ pgexporter = 3+ autovacuum workers = 3+ ─────────────────────────────────────────+ Total consumed = 41+ Free buffer = 56 (≈ 58 %)+```++The defaults are sized for **10 parallel flows** with concurrent API requests. If you run more flows or deploy multiple PentAGI instances against the same Postgres, raise `max_connections` via the `command` override in `docker-compose.yml` and increase the pool sizes proportionally:++```yaml+pgvector:+ image: vxcontrol/pgvector:latest+ command: postgres -c max_connections=200+```++To inspect the live connection budget on a running deployment:++```bash+# Postgres limits+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \+ "SELECT name, setting FROM pg_settings+ WHERE name IN ('"'"'max_connections'"'"', '"'"'superuser_reserved_connections'"'"');"'++# Current usage vs. available+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \+ "SELECT max_conn, used, max_conn - used AS available+ FROM (SELECT current_setting('"'"'max_connections'"'"')::int AS max_conn,+ count(*) AS used FROM pg_stat_activity) t;"'++# Breakdown by client+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \+ "SELECT application_name, client_addr, state, count(*)+ FROM pg_stat_activity+ WHERE pid <> pg_backend_pid()+ GROUP BY 1, 2, 3 ORDER BY count DESC;"'+```+#### Frontend ConfigurationEdit the configuration for `frontend` in `.vscode/launch.json` file:diff --git a/backend/cmd/pentagi/main.go b/backend/cmd/pentagi/main.goindex db6f67c5..1f1fec9c 100644--- a/backend/cmd/pentagi/main.go+++ b/backend/cmd/pentagi/main.go@@ -24,6 +24,7 @@ import (router "pentagi/pkg/server""pentagi/pkg/version"+ "github.com/jackc/pgx/v5/pgxpool"_ "github.com/lib/pq""github.com/pressly/goose/v3""github.com/sirupsen/logrus"@@ -80,17 +81,37 @@ func main() {log.Fatalf("Unable to open database: %v\n", err)}- db.SetMaxOpenConns(20)- db.SetMaxIdleConns(5)+ db.SetMaxOpenConns(cfg.DBMaxOpenConns)+ db.SetMaxIdleConns(cfg.DBMaxIdleConns)db.SetConnMaxLifetime(time.Hour)queries := database.New(db)- orm, err := database.NewGorm(cfg.DatabaseURL, "postgres")+ // Pass the same *sql.DB to GORM so both sqlc and GORM share one connection+ // pool. Previously each opened its own *sql.DB, consuming up to 40+ // Postgres connections. Now together they consume at most DBMaxOpenConns.+ orm, err := database.NewGorm(db, cfg.Debug)if err != nil {log.Fatalf("Unable to open database with gorm: %v\n", err)}+ // Create a shared pgxpool for all pgvector stores so that each executor+ // reuses pooled connections instead of opening a dedicated pgx.Connect.+ pgPoolConfig, err := pgxpool.ParseConfig(cfg.DatabaseURL)+ if err != nil {+ log.Fatalf("Failed to parse pgxpool config: %v\n", err)+ }+ pgPoolConfig.MaxConns = int32(cfg.DBVectorMaxConns)+ pgPool, err := pgxpool.NewWithConfig(ctx, pgPoolConfig)+ if err != nil {+ log.Fatalf("Failed to create pgxpool: %v\n", err)+ }+ defer pgPool.Close()++ // Attach the live pool to the config so router and tool executors can use+ // pgvector.WithConn(cfg.PgxPool) without any interface changes.+ cfg.PgxPool = pgPool+goose.SetBaseFS(migrations.EmbedMigrations)if err := goose.SetDialect("postgres"); err != nil {diff --git a/backend/docs/config.md b/backend/docs/config.mdindex ea9dd197..42b90ab5 100644--- a/backend/docs/config.md+++ b/backend/docs/config.md@@ -130,6 +130,9 @@ These settings control basic application behavior and are foundational for the s| Option | Environment Variable | Default Value | Description || -------------- | -------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------ || DatabaseURL | `DATABASE_URL` | `postgres://pentagiuser:pentagipass@pgvector:5432/pentagidb?sslmode=disable` | Connection string for the PostgreSQL database with pgvector extension |+| DBMaxOpenConns | `DB_MAX_OPEN_CONNS` | `25` | Maximum open connections in the shared `sql.DB` pool (sqlc + GORM combined). See [database.md §Connection Pooling](database.md#connection-pooling). |+| DBMaxIdleConns | `DB_MAX_IDLE_CONNS` | `5` | Maximum idle connections kept open between requests |+| DBVectorMaxConns | `DB_VECTOR_MAX_CONNS` | `10` | Maximum connections in the shared `pgxpool` for all pgvector stores || Debug | `DEBUG` | `false` | Enables debug mode with additional logging || DataDir | `DATA_DIR` | `./data` | Directory for storing persistent data || AskUser | `ASK_USER` | `false` | When enabled, requires explicit user confirmation for certain operations |@@ -138,21 +141,27 @@ These settings control basic application behavior and are foundational for the s### Usage Details-- **DatabaseURL**: This is a critical setting used throughout the application for all database connections. It's used to:- - Initialize the primary SQL database connection in `main.go`- - Create GORM ORM instances for model operations- - Configure pgvector connectivity for embedding operations- - Set up connection pools in various tools and executors+- **DatabaseURL**: This is a critical setting used throughout the application for all database connections. It is used to:+ - Initialize the single shared `sql.DB` connection pool in `main.go` (used by both sqlc `Queries` and GORM)+ - Seed the shared `pgxpool.Pool` for all pgvector stores (agent memory + knowledge API)++ For connection pool sizing and operational monitoring commands see [database.md §Connection Pooling](database.md#connection-pooling).```go-// In main.go for SQL connection+// In main.go — one pool shared by sqlc Queries and GORMdb, err := sql.Open("postgres", cfg.DatabaseURL)--// In main.go for GORM connection-orm, err := database.NewGorm(cfg.DatabaseURL, "postgres")--// In tools for vector database operations-pgvector.WithConnectionURL(fte.cfg.DatabaseURL)+db.SetMaxOpenConns(cfg.DBMaxOpenConns)+db.SetMaxIdleConns(cfg.DBMaxIdleConns)+db.SetConnMaxLifetime(time.Hour)++queries := database.New(db)+orm, err := database.NewGorm(db) // GORM wraps the same *sql.DB++// Shared pgxpool for all pgvector stores+pgPoolConfig, _ := pgxpool.ParseConfig(cfg.DatabaseURL)+pgPoolConfig.MaxConns = int32(cfg.DBVectorMaxConns)+pgPool, _ := pgxpool.NewWithConfig(ctx, pgPoolConfig)+cfg.PgxPool = pgPool // passed to tools and router```- **Debug**: Controls debug mode throughout the application, enabling additional logging and development features:diff --git a/backend/docs/database.md b/backend/docs/database.mdindex 943f94e3..1074eede 100644--- a/backend/docs/database.md+++ b/backend/docs/database.md@@ -1199,23 +1199,81 @@ return tx.Commit()### Connection Pooling-The package provides optimized connection pooling through GORM:+PentAGI opens two independent connection pools to the same Postgres instance:++| Pool | Env var | Default | Used by |+|---|---|---|---|+| Shared `sql.DB` | `DB_MAX_OPEN_CONNS` | `25` | sqlc `Queries` and GORM — both clients are backed by the **same** `*sql.DB` created in `main.go` |+| Shared `pgxpool` | `DB_VECTOR_MAX_CONNS` | `10` | All `pgvector.Store` instances: every flow/assistant tool executor + knowledge API |++Additional knob: `DB_MAX_IDLE_CONNS` (default `5`) — idle connections kept open between requests.++`NewGorm` accepts the already-configured `*sql.DB` so GORM never opens its own pool:+```go-func NewGorm(dsn, dbType string) (*gorm.DB, error) {- db, err := gorm.Open(dbType, dsn)- if err != nil {- return nil, err- }+// main.go — one pool, used by both sqlc and GORM+db.SetMaxOpenConns(cfg.DBMaxOpenConns)+db.SetMaxIdleConns(cfg.DBMaxIdleConns)+db.SetConnMaxLifetime(time.Hour)- // Optimized connection settings- db.DB().SetMaxIdleConns(5)- db.DB().SetMaxOpenConns(20)- db.DB().SetConnMaxLifetime(time.Hour)+orm, err := database.NewGorm(db)- return db, nil+// pkg/database/database.go+func NewGorm(db *sql.DB) (*gorm.DB, error) {+ orm, err := gorm.Open("postgres", db)+ ...}```+The shared `pgxpool` is created in `main.go` and stored in `cfg.PgxPool`. All pgvector stores+(tool executors via `SetEmbedder`, knowledge API in `router.go`) use `pgvector.WithConn(cfg.PgxPool)`+instead of opening individual `pgx.Connect` calls per executor.++**Connection budget** for the stock `vxcontrol/pgvector` image (`max_connections = 100`,+`superuser_reserved_connections = 3`):++```+Available for client connections = 97+ pentagi sql.DB (DB_MAX_OPEN_CONNS) = 25+ pentagi pgxpool (DB_VECTOR_MAX_CONNS) = 10+ pgexporter = 3+ autovacuum workers = 3+ ─────────────────────────────────────────────+ Total consumed = 41+ Free buffer = 56 (≈ 58%)+```++Defaults are sized for 10 parallel flows with concurrent API requests. To inspect the live+budget on a running deployment:++```bash+# Postgres limits+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \+ "SELECT name, setting FROM pg_settings+ WHERE name IN ('"'"'max_connections'"'"', '"'"'superuser_reserved_connections'"'"');"'++# Current usage vs. available+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \+ "SELECT max_conn, used, max_conn - used AS available+ FROM (SELECT current_setting('"'"'max_connections'"'"')::int AS max_conn,+ count(*) AS used FROM pg_stat_activity) t;"'++# Breakdown by client+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \+ "SELECT application_name, client_addr, state, count(*)+ FROM pg_stat_activity+ WHERE pid <> pg_backend_pid()+ GROUP BY 1, 2, 3 ORDER BY count DESC;"'+```++To raise the Postgres limit, add a `command` override in `docker-compose.yml`:++```yaml+pgvector:+ image: vxcontrol/pgvector:latest+ command: postgres -c max_connections=200+```+### Vector OperationsFor pgvector operations, consider:diff --git a/backend/pkg/config/config.go b/backend/pkg/config/config.goindex c40ba553..3f52212e 100644--- a/backend/pkg/config/config.go+++ b/backend/pkg/config/config.go@@ -10,6 +10,7 @@ import ("github.com/caarlos0/env/v10""github.com/google/uuid"+ "github.com/jackc/pgx/v5/pgxpool""github.com/joho/godotenv""github.com/vxcontrol/cloud/anonymizer/patterns""github.com/vxcontrol/cloud/sdk"@@ -223,6 +224,19 @@ type Config struct {// === Agent Planning Phase Configuration ===AgentPlanningStepEnabled bool `env:"AGENT_PLANNING_STEP_ENABLED" envDefault:"false"`++ // === Database Connection Pool Sizing ===+ // See backend/docs/database.md §Connection Pooling for budget calculation and+ // operational commands. Both sqlc (Queries) and GORM share a single *sql.DB,+ // so DB_MAX_OPEN_CONNS is the total sql.DB budget for the process.+ // Ensure DB_MAX_OPEN_CONNS + DB_VECTOR_MAX_CONNS < Postgres max_connections.+ DBMaxOpenConns int `env:"DB_MAX_OPEN_CONNS" envDefault:"25"`+ DBMaxIdleConns int `env:"DB_MAX_IDLE_CONNS" envDefault:"5"`+ DBVectorMaxConns int `env:"DB_VECTOR_MAX_CONNS" envDefault:"10"`++ // PgxPool is the shared pgxpool.Pool for all pgvector stores. Populated by+ // main after pool creation; NOT sourced from environment variables.+ PgxPool *pgxpool.Pool `env:"-"`}func NewConfig() (*Config, error) {diff --git a/backend/pkg/database/database.go b/backend/pkg/database/database.goindex 6e5893fe..96abc95a 100644--- a/backend/pkg/database/database.go+++ b/backend/pkg/database/database.go@@ -123,15 +123,16 @@ func (*GormLogger) Print(v ...interface{}) {}}-func NewGorm(dsn, dbType string) (*gorm.DB, error) {- db, err := gorm.Open(dbType, dsn)+// NewGorm creates a GORM instance backed by the provided *sql.DB.+// Passing an existing db ensures GORM shares the same connection pool as+// the sqlc Queries client, so the two clients together consume at most+// DBMaxOpenConns connections instead of doubling the Postgres load.+func NewGorm(db *sql.DB, debug bool) (*gorm.DB, error) {+ orm, err := gorm.Open("postgres", db)if err != nil {return nil, err}- db.DB().SetMaxIdleConns(5)- db.DB().SetMaxOpenConns(20)- db.DB().SetConnMaxLifetime(time.Hour)- db.SetLogger(&GormLogger{})- db.LogMode(true)- return db, nil+ orm.SetLogger(&GormLogger{})+ orm.LogMode(debug)+ return orm, nil}diff --git a/backend/pkg/server/router.go b/backend/pkg/server/router.goindex 4c685a1d..628f2c7c 100644--- a/backend/pkg/server/router.go+++ b/backend/pkg/server/router.go@@ -132,12 +132,16 @@ func NewRouter(embedder := providers.Embedder()var pgStore *pgvector.Storeif embedder.IsAvailable() {- if s, err := pgvector.New(- context.Background(),- pgvector.WithConnectionURL(cfg.DatabaseURL),+ opts := []pgvector.Option{pgvector.WithEmbedder(embedder),pgvector.WithCollectionName("langchain"),- ); err == nil {+ }+ if cfg.PgxPool != nil {+ opts = append(opts, pgvector.WithConn(cfg.PgxPool))+ } else {+ opts = append(opts, pgvector.WithConnectionURL(cfg.DatabaseURL))+ }+ if s, err := pgvector.New(context.Background(), opts...); err == nil {pgStore = &s} else {logrus.WithError(err).Warn("failed to initialise pgvector store for knowledge API; embedding operations will be unavailable")diff --git a/backend/pkg/tools/tools.go b/backend/pkg/tools/tools.goindex c025d4dd..13922e71 100644--- a/backend/pkg/tools/tools.go+++ b/backend/pkg/tools/tools.go@@ -382,15 +382,26 @@ func (fte *flowToolsExecutor) SetEmbedder(embedder embeddings.Embedder) {}if fte.store != nil {- fte.store.Close()+ // Do NOT close the store when it is backed by the shared pgxpool — calling+ // Close() on that store would terminate the pool for every other executor.+ // With a per-connection URL the store owns its connection and must close it.+ if fte.cfg.PgxPool == nil {+ fte.store.Close()+ }+ fte.store = nil}- store, err := pgvector.New(- context.Background(),- pgvector.WithConnectionURL(fte.cfg.DatabaseURL),+ opts := []pgvector.Option{pgvector.WithEmbedder(embedder),pgvector.WithCollectionName("langchain"),- )+ }+ if fte.cfg.PgxPool != nil {+ opts = append(opts, pgvector.WithConn(fte.cfg.PgxPool))+ } else {+ opts = append(opts, pgvector.WithConnectionURL(fte.cfg.DatabaseURL))+ }++ store, err := pgvector.New(context.Background(), opts...)if err == nil {fte.store = &store}@@ -682,7 +693,13 @@ func (fte *flowToolsExecutor) cachedResourcesDir() (string, error) {func (fte *flowToolsExecutor) Release(ctx context.Context) error {if fte.store != nil {- fte.store.Close()+ // Do NOT close the store when it is backed by the shared pgxpool — the pool+ // outlives individual flows and is shared by all executors. Only close when+ // the store owns its own connection (no shared pool configured).+ if fte.cfg.PgxPool == nil {+ fte.store.Close()+ }+ fte.store = nil}// TODO: here better to get flow containers list and purge all of them