Skip to content

Fully autonomous AI Agents system capable of performing complex penetration testing tasks

feat(database): enhance PostgreSQL connection pooling and configuration

Dmitry Ng077ddce476a6
diff --git a/README.md b/README.md
index b31b4039..5f7ef98d 100644
--- a/README.md
+++ b/README.md
@@ -2613,6 +2613,61 @@ Optional:
- `SERVER_PORT` - Port to run the server (default: `8443`)
- `SERVER_USE_SSL` - Enable SSL for the server (default: `false`)
+##### PostgreSQL / pgvector connection pool sizing
+
+PentAGI opens two independent connection pools to the same Postgres instance:
+
+| Pool | Env var | Default | Used by |
+|---|---|---|---|
+| Shared `sql.DB` | `DB_MAX_OPEN_CONNS` | `25` | All sqlc queries and GORM handlers share a single `*sql.DB` |
+| Shared `pgxpool` | `DB_VECTOR_MAX_CONNS` | `10` | All pgvector stores (agent memory + knowledge API) share a single pool |
+
+Additional tuning knob:
+- `DB_MAX_IDLE_CONNS` — maximum idle connections kept open in the `sql.DB` pool between requests (default: `5`).
+
+**Budget for the stock `vxcontrol/pgvector` image** (`max_connections = 100`, `superuser_reserved_connections = 3`):
+
+```
+Available for client connections = 97
+ pentagi sql.DB (DB_MAX_OPEN_CONNS) = 25
+ pentagi pgxpool (DB_VECTOR_MAX_CONNS) = 10
+ pgexporter = 3
+ autovacuum workers = 3
+ ─────────────────────────────────────────
+ Total consumed = 41
+ Free buffer = 56 (≈ 58 %)
+```
+
+The defaults are sized for **10 parallel flows** with concurrent API requests. If you run more flows or deploy multiple PentAGI instances against the same Postgres, raise `max_connections` via the `command` override in `docker-compose.yml` and increase the pool sizes proportionally:
+
+```yaml
+pgvector:
+ image: vxcontrol/pgvector:latest
+ command: postgres -c max_connections=200
+```
+
+To inspect the live connection budget on a running deployment:
+
+```bash
+# Postgres limits
+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
+ "SELECT name, setting FROM pg_settings
+ WHERE name IN ('"'"'max_connections'"'"', '"'"'superuser_reserved_connections'"'"');"'
+
+# Current usage vs. available
+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
+ "SELECT max_conn, used, max_conn - used AS available
+ FROM (SELECT current_setting('"'"'max_connections'"'"')::int AS max_conn,
+ count(*) AS used FROM pg_stat_activity) t;"'
+
+# Breakdown by client
+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
+ "SELECT application_name, client_addr, state, count(*)
+ FROM pg_stat_activity
+ WHERE pid <> pg_backend_pid()
+ GROUP BY 1, 2, 3 ORDER BY count DESC;"'
+```
+
#### Frontend Configuration
Edit the configuration for `frontend` in `.vscode/launch.json` file:
diff --git a/backend/cmd/pentagi/main.go b/backend/cmd/pentagi/main.go
index db6f67c5..1f1fec9c 100644
--- a/backend/cmd/pentagi/main.go
+++ b/backend/cmd/pentagi/main.go
@@ -24,6 +24,7 @@ import (
router "pentagi/pkg/server"
"pentagi/pkg/version"
+ "github.com/jackc/pgx/v5/pgxpool"
_ "github.com/lib/pq"
"github.com/pressly/goose/v3"
"github.com/sirupsen/logrus"
@@ -80,17 +81,37 @@ func main() {
log.Fatalf("Unable to open database: %v\n", err)
}
- db.SetMaxOpenConns(20)
- db.SetMaxIdleConns(5)
+ db.SetMaxOpenConns(cfg.DBMaxOpenConns)
+ db.SetMaxIdleConns(cfg.DBMaxIdleConns)
db.SetConnMaxLifetime(time.Hour)
queries := database.New(db)
- orm, err := database.NewGorm(cfg.DatabaseURL, "postgres")
+ // Pass the same *sql.DB to GORM so both sqlc and GORM share one connection
+ // pool. Previously each opened its own *sql.DB, consuming up to 40
+ // Postgres connections. Now together they consume at most DBMaxOpenConns.
+ orm, err := database.NewGorm(db, cfg.Debug)
if err != nil {
log.Fatalf("Unable to open database with gorm: %v\n", err)
}
+ // Create a shared pgxpool for all pgvector stores so that each executor
+ // reuses pooled connections instead of opening a dedicated pgx.Connect.
+ pgPoolConfig, err := pgxpool.ParseConfig(cfg.DatabaseURL)
+ if err != nil {
+ log.Fatalf("Failed to parse pgxpool config: %v\n", err)
+ }
+ pgPoolConfig.MaxConns = int32(cfg.DBVectorMaxConns)
+ pgPool, err := pgxpool.NewWithConfig(ctx, pgPoolConfig)
+ if err != nil {
+ log.Fatalf("Failed to create pgxpool: %v\n", err)
+ }
+ defer pgPool.Close()
+
+ // Attach the live pool to the config so router and tool executors can use
+ // pgvector.WithConn(cfg.PgxPool) without any interface changes.
+ cfg.PgxPool = pgPool
+
goose.SetBaseFS(migrations.EmbedMigrations)
if err := goose.SetDialect("postgres"); err != nil {
diff --git a/backend/docs/config.md b/backend/docs/config.md
index ea9dd197..42b90ab5 100644
--- a/backend/docs/config.md
+++ b/backend/docs/config.md
@@ -130,6 +130,9 @@ These settings control basic application behavior and are foundational for the s
| Option | Environment Variable | Default Value | Description |
| -------------- | -------------------- | ---------------------------------------------------------------------------- | ------------------------------------------------------------------------ |
| DatabaseURL | `DATABASE_URL` | `postgres://pentagiuser:pentagipass@pgvector:5432/pentagidb?sslmode=disable` | Connection string for the PostgreSQL database with pgvector extension |
+| DBMaxOpenConns | `DB_MAX_OPEN_CONNS` | `25` | Maximum open connections in the shared `sql.DB` pool (sqlc + GORM combined). See [database.md §Connection Pooling](database.md#connection-pooling). |
+| DBMaxIdleConns | `DB_MAX_IDLE_CONNS` | `5` | Maximum idle connections kept open between requests |
+| DBVectorMaxConns | `DB_VECTOR_MAX_CONNS` | `10` | Maximum connections in the shared `pgxpool` for all pgvector stores |
| Debug | `DEBUG` | `false` | Enables debug mode with additional logging |
| DataDir | `DATA_DIR` | `./data` | Directory for storing persistent data |
| AskUser | `ASK_USER` | `false` | When enabled, requires explicit user confirmation for certain operations |
@@ -138,21 +141,27 @@ These settings control basic application behavior and are foundational for the s
### Usage Details
-- **DatabaseURL**: This is a critical setting used throughout the application for all database connections. It's used to:
- - Initialize the primary SQL database connection in `main.go`
- - Create GORM ORM instances for model operations
- - Configure pgvector connectivity for embedding operations
- - Set up connection pools in various tools and executors
+- **DatabaseURL**: This is a critical setting used throughout the application for all database connections. It is used to:
+ - Initialize the single shared `sql.DB` connection pool in `main.go` (used by both sqlc `Queries` and GORM)
+ - Seed the shared `pgxpool.Pool` for all pgvector stores (agent memory + knowledge API)
+
+ For connection pool sizing and operational monitoring commands see [database.md §Connection Pooling](database.md#connection-pooling).
```go
-// In main.go for SQL connection
+// In main.go — one pool shared by sqlc Queries and GORM
db, err := sql.Open("postgres", cfg.DatabaseURL)
-
-// In main.go for GORM connection
-orm, err := database.NewGorm(cfg.DatabaseURL, "postgres")
-
-// In tools for vector database operations
-pgvector.WithConnectionURL(fte.cfg.DatabaseURL)
+db.SetMaxOpenConns(cfg.DBMaxOpenConns)
+db.SetMaxIdleConns(cfg.DBMaxIdleConns)
+db.SetConnMaxLifetime(time.Hour)
+
+queries := database.New(db)
+orm, err := database.NewGorm(db) // GORM wraps the same *sql.DB
+
+// Shared pgxpool for all pgvector stores
+pgPoolConfig, _ := pgxpool.ParseConfig(cfg.DatabaseURL)
+pgPoolConfig.MaxConns = int32(cfg.DBVectorMaxConns)
+pgPool, _ := pgxpool.NewWithConfig(ctx, pgPoolConfig)
+cfg.PgxPool = pgPool // passed to tools and router
```
- **Debug**: Controls debug mode throughout the application, enabling additional logging and development features:
diff --git a/backend/docs/database.md b/backend/docs/database.md
index 943f94e3..1074eede 100644
--- a/backend/docs/database.md
+++ b/backend/docs/database.md
@@ -1199,23 +1199,81 @@ return tx.Commit()
### Connection Pooling
-The package provides optimized connection pooling through GORM:
+PentAGI opens two independent connection pools to the same Postgres instance:
+
+| Pool | Env var | Default | Used by |
+|---|---|---|---|
+| Shared `sql.DB` | `DB_MAX_OPEN_CONNS` | `25` | sqlc `Queries` and GORM — both clients are backed by the **same** `*sql.DB` created in `main.go` |
+| Shared `pgxpool` | `DB_VECTOR_MAX_CONNS` | `10` | All `pgvector.Store` instances: every flow/assistant tool executor + knowledge API |
+
+Additional knob: `DB_MAX_IDLE_CONNS` (default `5`) — idle connections kept open between requests.
+
+`NewGorm` accepts the already-configured `*sql.DB` so GORM never opens its own pool:
+
```go
-func NewGorm(dsn, dbType string) (*gorm.DB, error) {
- db, err := gorm.Open(dbType, dsn)
- if err != nil {
- return nil, err
- }
+// main.go — one pool, used by both sqlc and GORM
+db.SetMaxOpenConns(cfg.DBMaxOpenConns)
+db.SetMaxIdleConns(cfg.DBMaxIdleConns)
+db.SetConnMaxLifetime(time.Hour)
- // Optimized connection settings
- db.DB().SetMaxIdleConns(5)
- db.DB().SetMaxOpenConns(20)
- db.DB().SetConnMaxLifetime(time.Hour)
+orm, err := database.NewGorm(db)
- return db, nil
+// pkg/database/database.go
+func NewGorm(db *sql.DB) (*gorm.DB, error) {
+ orm, err := gorm.Open("postgres", db)
+ ...
}
```
+The shared `pgxpool` is created in `main.go` and stored in `cfg.PgxPool`. All pgvector stores
+(tool executors via `SetEmbedder`, knowledge API in `router.go`) use `pgvector.WithConn(cfg.PgxPool)`
+instead of opening individual `pgx.Connect` calls per executor.
+
+**Connection budget** for the stock `vxcontrol/pgvector` image (`max_connections = 100`,
+`superuser_reserved_connections = 3`):
+
+```
+Available for client connections = 97
+ pentagi sql.DB (DB_MAX_OPEN_CONNS) = 25
+ pentagi pgxpool (DB_VECTOR_MAX_CONNS) = 10
+ pgexporter = 3
+ autovacuum workers = 3
+ ─────────────────────────────────────────────
+ Total consumed = 41
+ Free buffer = 56 (≈ 58%)
+```
+
+Defaults are sized for 10 parallel flows with concurrent API requests. To inspect the live
+budget on a running deployment:
+
+```bash
+# Postgres limits
+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
+ "SELECT name, setting FROM pg_settings
+ WHERE name IN ('"'"'max_connections'"'"', '"'"'superuser_reserved_connections'"'"');"'
+
+# Current usage vs. available
+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
+ "SELECT max_conn, used, max_conn - used AS available
+ FROM (SELECT current_setting('"'"'max_connections'"'"')::int AS max_conn,
+ count(*) AS used FROM pg_stat_activity) t;"'
+
+# Breakdown by client
+docker exec pgvector sh -c 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c \
+ "SELECT application_name, client_addr, state, count(*)
+ FROM pg_stat_activity
+ WHERE pid <> pg_backend_pid()
+ GROUP BY 1, 2, 3 ORDER BY count DESC;"'
+```
+
+To raise the Postgres limit, add a `command` override in `docker-compose.yml`:
+
+```yaml
+pgvector:
+ image: vxcontrol/pgvector:latest
+ command: postgres -c max_connections=200
+```
+
### Vector Operations
For pgvector operations, consider:
diff --git a/backend/pkg/config/config.go b/backend/pkg/config/config.go
index c40ba553..3f52212e 100644
--- a/backend/pkg/config/config.go
+++ b/backend/pkg/config/config.go
@@ -10,6 +10,7 @@ import (
"github.com/caarlos0/env/v10"
"github.com/google/uuid"
+ "github.com/jackc/pgx/v5/pgxpool"
"github.com/joho/godotenv"
"github.com/vxcontrol/cloud/anonymizer/patterns"
"github.com/vxcontrol/cloud/sdk"
@@ -223,6 +224,19 @@ type Config struct {
// === Agent Planning Phase Configuration ===
AgentPlanningStepEnabled bool `env:"AGENT_PLANNING_STEP_ENABLED" envDefault:"false"`
+
+ // === Database Connection Pool Sizing ===
+ // See backend/docs/database.md §Connection Pooling for budget calculation and
+ // operational commands. Both sqlc (Queries) and GORM share a single *sql.DB,
+ // so DB_MAX_OPEN_CONNS is the total sql.DB budget for the process.
+ // Ensure DB_MAX_OPEN_CONNS + DB_VECTOR_MAX_CONNS < Postgres max_connections.
+ DBMaxOpenConns int `env:"DB_MAX_OPEN_CONNS" envDefault:"25"`
+ DBMaxIdleConns int `env:"DB_MAX_IDLE_CONNS" envDefault:"5"`
+ DBVectorMaxConns int `env:"DB_VECTOR_MAX_CONNS" envDefault:"10"`
+
+ // PgxPool is the shared pgxpool.Pool for all pgvector stores. Populated by
+ // main after pool creation; NOT sourced from environment variables.
+ PgxPool *pgxpool.Pool `env:"-"`
}
func NewConfig() (*Config, error) {
diff --git a/backend/pkg/database/database.go b/backend/pkg/database/database.go
index 6e5893fe..96abc95a 100644
--- a/backend/pkg/database/database.go
+++ b/backend/pkg/database/database.go
@@ -123,15 +123,16 @@ func (*GormLogger) Print(v ...interface{}) {
}
}
-func NewGorm(dsn, dbType string) (*gorm.DB, error) {
- db, err := gorm.Open(dbType, dsn)
+// NewGorm creates a GORM instance backed by the provided *sql.DB.
+// Passing an existing db ensures GORM shares the same connection pool as
+// the sqlc Queries client, so the two clients together consume at most
+// DBMaxOpenConns connections instead of doubling the Postgres load.
+func NewGorm(db *sql.DB, debug bool) (*gorm.DB, error) {
+ orm, err := gorm.Open("postgres", db)
if err != nil {
return nil, err
}
- db.DB().SetMaxIdleConns(5)
- db.DB().SetMaxOpenConns(20)
- db.DB().SetConnMaxLifetime(time.Hour)
- db.SetLogger(&GormLogger{})
- db.LogMode(true)
- return db, nil
+ orm.SetLogger(&GormLogger{})
+ orm.LogMode(debug)
+ return orm, nil
}
diff --git a/backend/pkg/server/router.go b/backend/pkg/server/router.go
index 4c685a1d..628f2c7c 100644
--- a/backend/pkg/server/router.go
+++ b/backend/pkg/server/router.go
@@ -132,12 +132,16 @@ func NewRouter(
embedder := providers.Embedder()
var pgStore *pgvector.Store
if embedder.IsAvailable() {
- if s, err := pgvector.New(
- context.Background(),
- pgvector.WithConnectionURL(cfg.DatabaseURL),
+ opts := []pgvector.Option{
pgvector.WithEmbedder(embedder),
pgvector.WithCollectionName("langchain"),
- ); err == nil {
+ }
+ if cfg.PgxPool != nil {
+ opts = append(opts, pgvector.WithConn(cfg.PgxPool))
+ } else {
+ opts = append(opts, pgvector.WithConnectionURL(cfg.DatabaseURL))
+ }
+ if s, err := pgvector.New(context.Background(), opts...); err == nil {
pgStore = &s
} else {
logrus.WithError(err).Warn("failed to initialise pgvector store for knowledge API; embedding operations will be unavailable")
diff --git a/backend/pkg/tools/tools.go b/backend/pkg/tools/tools.go
index c025d4dd..13922e71 100644
--- a/backend/pkg/tools/tools.go
+++ b/backend/pkg/tools/tools.go
@@ -382,15 +382,26 @@ func (fte *flowToolsExecutor) SetEmbedder(embedder embeddings.Embedder) {
}
if fte.store != nil {
- fte.store.Close()
+ // Do NOT close the store when it is backed by the shared pgxpool — calling
+ // Close() on that store would terminate the pool for every other executor.
+ // With a per-connection URL the store owns its connection and must close it.
+ if fte.cfg.PgxPool == nil {
+ fte.store.Close()
+ }
+ fte.store = nil
}
- store, err := pgvector.New(
- context.Background(),
- pgvector.WithConnectionURL(fte.cfg.DatabaseURL),
+ opts := []pgvector.Option{
pgvector.WithEmbedder(embedder),
pgvector.WithCollectionName("langchain"),
- )
+ }
+ if fte.cfg.PgxPool != nil {
+ opts = append(opts, pgvector.WithConn(fte.cfg.PgxPool))
+ } else {
+ opts = append(opts, pgvector.WithConnectionURL(fte.cfg.DatabaseURL))
+ }
+
+ store, err := pgvector.New(context.Background(), opts...)
if err == nil {
fte.store = &store
}
@@ -682,7 +693,13 @@ func (fte *flowToolsExecutor) cachedResourcesDir() (string, error) {
func (fte *flowToolsExecutor) Release(ctx context.Context) error {
if fte.store != nil {
- fte.store.Close()
+ // Do NOT close the store when it is backed by the shared pgxpool — the pool
+ // outlives individual flows and is shared by all executors. Only close when
+ // the store owns its own connection (no shared pool configured).
+ if fte.cfg.PgxPool == nil {
+ fte.store.Close()
+ }
+ fte.store = nil
}
// TODO: here better to get flow containers list and purge all of them