Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

Add LLM Monitoring Data retention policy to `DOKU_LLM_DATA` Table (#48)

Patcher991e040b65923a
diff --git a/src/ingester/config/config.go b/src/ingester/config/config.go
index 02789106..85b183ad 100644
--- a/src/ingester/config/config.go
+++ b/src/ingester/config/config.go
@@ -10,8 +10,9 @@ import (
)
type Configuration struct {
- IngesterPort string `yaml:"ingesterPort"`
- PricingInfo struct {
+ IngesterPort string `yaml:"ingesterPort"`
+ RentionPeriod string `yaml:"RetentionPeriod"`
+ PricingInfo struct {
LocalFile struct {
Path string `yaml:"path"`
} `yaml:"localFile"`
@@ -53,6 +54,9 @@ func validateConfig(cfg *Configuration) error {
if _, err := strconv.Atoi(cfg.IngesterPort); err != nil {
return fmt.Errorf("Ingester Port is not defined")
}
+ if _, err := strconv.Atoi(cfg.RentionPeriod); err != nil {
+ cfg.RentionPeriod = "6 months"
+ }
// Check if at least one PricingInfo configuration is set.
if cfg.PricingInfo.LocalFile.Path == "" && cfg.PricingInfo.URL == "" {
diff --git a/src/ingester/db/db.go b/src/ingester/db/db.go
index 0042643c..d76fbcdf 100644
--- a/src/ingester/db/db.go
+++ b/src/ingester/db/db.go
@@ -159,7 +159,7 @@ func tableExists(db *sql.DB, tableName string) (bool, error) {
}
// createTable creates a table in the database if it doesn't exist.
-func createTable(db *sql.DB, tableName string) error {
+func createTable(db *sql.DB, tableName string, retentionPeriod string) error {
var createTableSQL string
if tableName == "DOKU_APIKEYS" {
createTableSQL = getCreateAPIKeysTableSQL(tableName)
@@ -194,6 +194,12 @@ func createTable(db *sql.DB, tableName string) error {
return fmt.Errorf("Error creating hypertable: %w", err)
}
log.Info().Msgf("Table '%s' converted to a Hypertable", tableName)
+ query := fmt.Sprintf("SELECT add_retention_policy('%s', INTERVAL '%s')", tableName, retentionPeriod)
+ _, err = db.Exec(query)
+ if err != nil {
+ return fmt.Errorf("Error adding data retention policy: %w", err)
+ }
+ log.Info().Msgf("Added data retention policy of '%s' to '%s' ", retentionPeriod, tableName)
}
} else {
log.Info().Msgf("Table '%s' already exists in the database", tableName)
@@ -321,13 +327,13 @@ func Init(cfg config.Configuration) error {
// Create the DATA and API keys table if it doesn't exist.
log.Info().Msgf("Creating '%s' and '%s' tables in the database if they don't exist", "DOKU_APIKEYS", "DOKU_LLM_DATA")
- err = createTable(db, "DOKU_APIKEYS")
+ err = createTable(db, "DOKU_APIKEYS", cfg.RentionPeriod)
if err != nil {
log.Error().Err(err).Msgf("Error creating table %s", "DOKU_APIKEYS")
return err
}
- err = createTable(db, "DOKU_LLM_DATA")
+ err = createTable(db, "DOKU_LLM_DATA", cfg.RentionPeriod)
if err != nil {
log.Error().Err(err).Msgf("Error creating table %s", "DOKU_APIKEYS")
return err