Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

Add Ingester

patcher99d8072a82126e
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
new file mode 100644
index 00000000..737811a0
--- /dev/null
+++ b/.github/workflows/docker-publish.yml
@@ -0,0 +1,114 @@
+name: Release
+
+on:
+ push:
+ tags: [ '*.*.*' ]
+
+env:
+ REGISTRY: ghcr.io
+ IMAGE_NAME: ${{ github.repository }}
+
+permissions:
+ contents: write
+
+jobs:
+ docker:
+ runs-on: ubuntu-latest
+ outputs:
+ version: ${{ steps.set-version-output.outputs.version }}
+ permissions:
+ contents: write
+ packages: write
+ id-token: write
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Install cosign
+ uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1
+ with:
+ cosign-release: 'v2.1.1'
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+ with:
+ platforms: linux/arm64/v8,linux/arm/v8
+
+ - name: Code Format Check (gofmt)
+ run: |
+ if [ -n "$(gofmt -l .)" ]; then
+ echo "Go code is not formatted:"
+ gofmt -d .
+ exit 1
+ fi
+
+ - name: Log into registry ${{ env.REGISTRY }}
+ uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+ with:
+ registry: ${{ env.REGISTRY }}
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Extract Docker metadata
+ id: meta
+ uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+ with:
+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+ - name: Build and push Docker image
+ id: build-and-push
+ uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+ with:
+ context: .
+ push: ${{ github.event_name != 'pull_request' }}
+ tags: ${{ steps.meta.outputs.tags }}
+ labels: ${{ steps.meta.outputs.labels }}
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
+
+ - name: Sign the published Docker image
+ env:
+ TAGS: ${{ steps.meta.outputs.tags }}
+ DIGEST: ${{ steps.build-and-push.outputs.digest }}
+ run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
+
+ - name: Set version output
+ id: set-version-output
+ run: echo "::set-output name=version::${{ env.DOCKER_METADATA_OUTPUT_VERSION }}"
+
+ binaries:
+ name: Publish binaries
+ needs: docker
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ platform:
+ - {os: "linux", arch: "amd64"}
+ - {os: "linux", arch: "arm64"}
+ - {os: "linux", arch: "arm"}
+ - {os: "linux", arch: "ppc64le"}
+ - {os: "windows", arch: "amd64"}
+ - {os: "windows", arch: "386"}
+ - {os: "darwin", arch: "amd64"}
+ - {os: "darwin", arch: "arm64"}
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - uses: actions/setup-go@v4
+ with:
+ go-version: '1.21.5'
+
+ - name: Build package for ${{ matrix.platform.os }}-${{ matrix.platform.arch }}
+ working-directory: ./src
+ run: env GOOS=${{ matrix.platform.os }} GOARCH=${{ matrix.platform.arch }} go build -o ingester-${{ matrix.platform.os }}-${{ matrix.platform.arch }}
+
+ - name: Upload binaries to release
+ uses: svenstaro/upload-release-action@v2
+ with:
+ repo_token: ${{ secrets.GITHUB_TOKEN }}
+ file: src/ingester-${{ matrix.platform.os }}-${{ matrix.platform.arch }}
+ asset_name: ingester-${{ matrix.platform.os }}-${{ matrix.platform.arch }}
+ tag: ${{ needs.docker.outputs.version }}
+ overwrite: true
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 00000000..ddea41be
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,37 @@
+name: Tests
+
+on:
+ push:
+ branches: [ "main" ]
+ pull_request:
+ branches: [ "main" ]
+
+defaults:
+ run:
+ working-directory: src
+
+jobs:
+
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up Go
+ uses: actions/setup-go@v4
+ with:
+ go-version: '1.21.1'
+
+ - name: Build
+ run: go build -v ./...
+
+ - name: Code Format Check (gofmt)
+ run: |
+ if [ -n "$(gofmt -l .)" ]; then
+ echo "Go code is not formatted:"
+ gofmt -d .
+ exit 1
+ fi
+
+ - name: Test
+ run: go test -v ./...
diff --git a/.gitignore b/.gitignore
index b512c09d..610c8034 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
-node_modules
\ No newline at end of file
+node_modules
+test.py
+test.go
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..c7c2098b
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,17 @@
+# Use the official Go image as the base image
+FROM golang:1.21.5
+
+# Set the working directory to /app
+WORKDIR /app
+
+COPY src/go.mod .
+COPY src/go.sum .
+
+# Download dependencies in a separate layer to leverage Docker cache
+RUN go mod download
+
+# Now copy the rest of your source code.
+COPY src .
+
+# Build the Go application and name the executable as "ingester"
+RUN go build -o ingester
\ No newline at end of file
diff --git a/assets/example-config.yml b/assets/example-config.yml
new file mode 100644
index 00000000..c2d1ac8c
--- /dev/null
+++ b/assets/example-config.yml
@@ -0,0 +1,41 @@
+ingesterPort: 9044 # Port to run the Ingestion Server on, Example: 9044
+
+# Comment out either 'localFile' or 'url' to use only one at a time.
+pricingInfo:
+ # localFile:
+ # path: "/assets/pricing.json" # Path to local JSON file with LLM Pricing data
+ url: "https://raw.githubusercontent.com/dokulabs/ingester/main/assets/pricing.json" # URL to download Pricing data file
+
+# Configuration for the Doku Backend Database (TimescaleDB)
+dbConfig:
+ name: "DBNAME" # Name of the database, Example: "postgres"
+ username: "USERNAME" # Username to connect to the database, Example: "admin"
+ password: "PASSWORD" # Password to connect to the database, Example: "DOKU"
+ host: "HOSTURL" # Hostname of the database, Example: "174.21.1.2"
+ port: DBPORT # Port of the database, Example: 5432
+ sslMode: SSLMODE # SSL Mode of the database, Example: "require"
+ maxOpenConns: MAXOPENCONNS # Maximum number of open connections to the database, Example: 15
+ maxIdleConns: MAXOPENCONNS # Maximum number of idle connections to the database, Example: 15
+ dataTable: DATATABLE # Name of the table to store LLM Data, Example: "DOKU"
+ apiKeyTable: APIKEYTABLE # Name of the table to store API Keys, Example: "APIKEYS"
+
+# Configure Platform to export LLM Observability Data from Doku
+# Only one platform can be enabled at a time, To enable a platform, set enabled to true and fill in the required fields for that platform.
+observabilityPlatform:
+ enabled: false # Enable or Disable the Observability Platform, Example: true
+ # grafanaCloud:
+ # promUrl: "influx-line-proxy-url" # URL to the Influx Line Proxy URL of the Grafana Cloud Prometheus Instance
+ # promUsername: "prometheus-userid" # Prometheus User ID of the Grafana Cloud Prometheus Instance
+ # lokiUrl: "loki-push-url" # URL to the Loki Push URL of the Grafana Cloud Loki Instance
+ # lokiUsername: "loki-username" # Loki Username of the Grafana Cloud Loki Instance
+ # accessToken: "grafana-cloud-access-token" # Access Token of the Grafana Cloud Instance
+
+ # newRelic:
+ # metricsUrl: "https://metric-api.newrelic.com/metric/v1" # URL to the New Relic Metric API
+ # logsUrl: "https://log-api.newrelic.com/log/v1" # URL to the New Relic Log API
+ # key: "newrelic-api-key" # Ingest API Key of the New Relic Account
+
+ # datadog:
+ # metricsUrl: "https://api.datadoghq.com/api/v1/series" # URL to the DataDog Metrics API
+ # logsUrl: "https://http.logs.datadoghq.com/api/v2/logs" # URL to the DataDog Logs API
+ # apiKey: "datadog-api-key" # API Key of the DataDog Account
\ No newline at end of file
diff --git a/assets/pricing.json b/assets/pricing.json
new file mode 100644
index 00000000..a92d3830
--- /dev/null
+++ b/assets/pricing.json
@@ -0,0 +1,145 @@
+{
+ "embeddings": {
+ "text-embedding-ada-002": 0.0001,
+ "ada": 0.0001,
+ "text-ada-001": 0.0001,
+ "embed-english-v3.0": 0.0001,
+ "embed-multilingual-v3.0": 0.0001,
+ "embed-english-light-v3.0": 0.0001,
+ "embed-multilingual-light-v3.0": 0.0001,
+ "embed-english-v2.0": 0.0001,
+ "embed-english-light-v2.0": 0.0001,
+ "embed-multilingual-v2.0": 0.0001
+ },
+ "images": {
+ "dall-e-3": {
+ "standard": {
+ "1024x1024": 0.040,
+ "1024x1792": 0.020,
+ "1792x1024": 0.080
+ },
+ "hd": {
+ "1024x1024": 0.080,
+ "1024x1792": 0.120,
+ "1792x1024": 0.120
+ }
+ },
+ "dall-e-2": {
+ "standard": {
+ "1024x1024": 0.020,
+ "512x512": 0.018,
+ "256x256": 0.016
+ }
+ }
+ },
+ "audio": {
+ "tts-1" : 0.015,
+ "tts-1-hd": 0.030
+ },
+ "chat": {
+ "ada": {
+ "promptPrice": 0.0004,
+ "completionPrice": 0.0004
+ },
+ "text-ada-001": {
+ "promptPrice": 0.0004,
+ "completionPrice": 0.0004
+ },
+ "babbage": {
+ "promptPrice": 0.0005,
+ "completionPrice": 0.0005
+ },
+ "babbage-002": {
+ "promptPrice": 0.0004,
+ "completionPrice": 0.0004
+ },
+ "text-babbage-001": {
+ "promptPrice": 0.0005,
+ "completionPrice": 0.0005
+ },
+ "curie": {
+ "promptPrice": 0.0020,
+ "completionPrice": 0.0020
+ },
+ "text-curie-001": {
+ "promptPrice": 0.0020,
+ "completionPrice": 0.0020
+ },
+ "davinci": {
+ "promptPrice": 0.0200,
+ "completionPrice": 0.0200
+ },
+ "davinci-002": {
+ "promptPrice": 0.0020,
+ "completionPrice": 0.0020
+ },
+ "text-davinci-001": {
+ "promptPrice": 0.0200,
+ "completionPrice": 0.0200
+ },
+ "text-davinci-002": {
+ "promptPrice": 0.0200,
+ "completionPrice": 0.0200
+ },
+ "text-davinci-003": {
+ "promptPrice": 0.0200,
+ "completionPrice": 0.0200
+ },
+ "gpt-3.5-turbo": {
+ "promptPrice": 0.0010,
+ "completionPrice": 0.0020
+ },
+ "gpt-3.5-turbo-instruct": {
+ "promptPrice": 0.0015,
+ "completionPrice": 0.0020
+ },
+ "gpt-4": {
+ "promptPrice": 0.03,
+ "completionPrice": 0.06
+ },
+ "gpt-4-32k": {
+ "promptPrice": 0.06,
+ "completionPrice": 0.12
+ },
+ "gpt-4-1106-preview": {
+ "promptPrice": 0.01,
+ "completionPrice": 0.03
+ },
+ "gpt-4-preview": {
+ "promptPrice": 0.01,
+ "completionPrice": 0.03
+ },
+ "gpt-4-1106-vision-preview": {
+ "promptPrice": 0.01,
+ "completionPrice": 0.03
+ },
+ "gpt-4-vision-preview": {
+ "promptPrice": 0.01,
+ "completionPrice": 0.03
+ },
+ "claude-instant-1": {
+ "promptPrice": 0.00163,
+ "completionPrice": 0.00551
+ },
+ "claude-2": {
+ "promptPrice": 0.01102,
+ "completionPrice": 0.03268
+ },
+ "command": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "command-nightly": {
+ "promptPrice": 0.001,
+ "completionPrice": 0.002
+ },
+ "command-light": {
+ "promptPrice": 0.0003,
+ "completionPrice": 0.0006
+ },
+ "command-light-nightly": {
+ "promptPrice": 0.0003,
+ "completionPrice": 0.0006
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/ingester/api/handlers.go b/src/ingester/api/handlers.go
new file mode 100644
index 00000000..389ed828
--- /dev/null
+++ b/src/ingester/api/handlers.go
@@ -0,0 +1,190 @@
+package api
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strings"
+
+ "ingester/auth"
+ "ingester/db"
+
+ "github.com/rs/zerolog/log"
+)
+
+const (
+ // Constants for error messages
+ errMsgKeyExists = "An API Key with the name '%s' already exists"
+ errMsgAuthFailed = "Unauthorized: Please check your API Key and try again"
+ errMsgKeyNotFound = "Unable to find API Key with the given name %s"
+ errMsgInvalidBody = "Invalid request body"
+)
+
+// APIKeyRequest represents the expected request structure for API Key related endpoints.
+type APIKeyRequest struct {
+ Name string `json:"name"`
+}
+
+// jsonResponse represents the expected response structure for all endpoints.
+type jsonResponse struct {
+ Status int `json:"status"`
+ Message string `json:"message"`
+ Data interface{} `json:"data,omitempty"`
+}
+
+// Normalize capitalizes names fields in the request body.
+func (r *APIKeyRequest) Normalize() {
+ r.Name = strings.ToLower(r.Name)
+}
+
+// decodeRequestBody decodes the JSON request body into the destination struct.
+func decodeRequestBody(r *http.Request, dest interface{}) error {
+ return json.NewDecoder(r.Body).Decode(dest)
+}
+
+// getAuthKey retrieves the API Key from the request header.
+func getAuthKey(r *http.Request) string {
+ return r.Header.Get("Authorization")
+}
+
+// sendJSONResponse constructs and sends a JSON response with appropriate headers.
+func sendJSONResponse(w http.ResponseWriter, status int, message string) {
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(status)
+
+ response := jsonResponse{
+ Status: status,
+ Message: message,
+ }
+
+ json.NewEncoder(w).Encode(response)
+}
+
+// handleAPIKeyErrors centralizes the error handling logic for API Key operations.
+func handleAPIKeyErrors(w http.ResponseWriter, err error, name string) {
+ if err.Error() == "KEYEXISTS" {
+ sendJSONResponse(w, http.StatusConflict, fmt.Sprintf(errMsgKeyExists, name))
+ return
+ } else if err.Error() == "AUTHFAILED" {
+ sendJSONResponse(w, http.StatusUnauthorized, errMsgAuthFailed)
+ return
+ } else if err.Error() == "NOTFOUND" {
+ sendJSONResponse(w, http.StatusNotFound, fmt.Sprintf(errMsgKeyNotFound, name))
+ return
+ } else {
+ sendJSONResponse(w, http.StatusInternalServerError, err.Error())
+ return
+ }
+}
+
+// generateAPIKeyHandler handles the creation of a new API Key
+func generateAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
+ var request APIKeyRequest
+
+ if err := decodeRequestBody(r, &request); err != nil {
+ sendJSONResponse(w, http.StatusBadRequest, errMsgInvalidBody)
+ return
+ }
+ request.Normalize()
+
+ newAPIKey, err := db.GenerateAPIKey(getAuthKey(r), request.Name)
+ if err != nil {
+ handleAPIKeyErrors(w, err, request.Name)
+ return
+ }
+
+ sendJSONResponse(w, http.StatusOK, newAPIKey)
+}
+
+// getAPIKeyHandler handles retrieving an existing API key.
+func getAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
+ var request APIKeyRequest
+
+ if err := decodeRequestBody(r, &request); err != nil {
+ sendJSONResponse(w, http.StatusBadRequest, errMsgInvalidBody)
+ return
+ }
+ request.Normalize()
+
+ apiKey, err := db.GetAPIKeyForName(getAuthKey(r), request.Name)
+ if err != nil {
+ handleAPIKeyErrors(w, err, request.Name)
+ return
+ }
+
+ sendJSONResponse(w, http.StatusOK, apiKey)
+}
+
+// deleteAPIKeyHandler handles deleting an existing API key.
+func deleteAPIKeyHandler(w http.ResponseWriter, r *http.Request) {
+ var request APIKeyRequest
+
+ if err := decodeRequestBody(r, &request); err != nil {
+ sendJSONResponse(w, http.StatusBadRequest, errMsgInvalidBody)
+ return
+ }
+
+ request.Normalize()
+ err := db.DeleteAPIKey(getAuthKey(r), request.Name)
+ if err != nil {
+ handleAPIKeyErrors(w, err, request.Name)
+ return
+ }
+
+ sendJSONResponse(w, http.StatusOK, "API key deleted successfully")
+}
+
+// DataHandler handles data related operations recieved on `/api/push` endpoint.
+func DataHandler(w http.ResponseWriter, r *http.Request) {
+ var data map[string]interface{}
+
+ if err := decodeRequestBody(r, &data); err != nil {
+ sendJSONResponse(w, http.StatusBadRequest, errMsgInvalidBody)
+ return
+ }
+
+ var err error
+ data["name"], err = auth.AuthenticateRequest(getAuthKey(r))
+ if err != nil {
+ handleAPIKeyErrors(w, err, "")
+ return
+ }
+
+ // Check if skipResp is true
+ skipResp, ok := data["skipResp"].(bool)
+ if ok && skipResp == true {
+ sendJSONResponse(w, http.StatusAccepted, "Insertion started in background")
+ go db.PerformDatabaseInsertion(data) // Running as a goroutine for async processing
+ return
+ }
+
+ responseMessage, statusCode := db.PerformDatabaseInsertion(data)
+
+ // Respond to the user with the insertion status and any potential errors
+ sendJSONResponse(w, statusCode, responseMessage)
+}
+
+// APIKeyHandler handles all API Key tasks recieved on `/api/keys` endpoint.
+func APIKeyHandler(w http.ResponseWriter, r *http.Request) {
+ switch r.Method {
+ case "GET":
+ getAPIKeyHandler(w, r)
+ case "POST":
+ generateAPIKeyHandler(w, r)
+ case "DELETE":
+ deleteAPIKeyHandler(w, r)
+ default:
+ w.WriteHeader(http.StatusMethodNotAllowed)
+ }
+}
+
+// BaseEndpoint serves as a health check and entry point for the service.
+func BaseEndpoint(w http.ResponseWriter, r *http.Request) {
+ if err := db.PingDB(); err != nil {
+ log.Error().Err(err).Msgf("Health check failed") // Log the error
+ sendJSONResponse(w, http.StatusServiceUnavailable, "Database is currently not reachable from the server")
+ return
+ }
+ // The database is up and reachable.
+ sendJSONResponse(w, http.StatusOK, "Welcome to Doku Ingester - Service operational")
+}
diff --git a/src/ingester/auth/auth.go b/src/ingester/auth/auth.go
new file mode 100644
index 00000000..102da29a
--- /dev/null
+++ b/src/ingester/auth/auth.go
@@ -0,0 +1,78 @@
+package auth
+
+import (
+ "database/sql"
+ "fmt"
+ "sync"
+ "time"
+
+ "ingester/db"
+
+ "github.com/rs/zerolog/log"
+)
+
+var (
+ // apiKeyCache stores the lookup of API keys and organization IDs.
+ apiKeyCache = sync.Map{}
+ // CacheEntryDuration defines how long an item should stay in the cache before being re-validated.
+ CacheEntryDuration = time.Minute * 10
+)
+
+// cacheEntry represents an entry in the API key cache.
+type cacheEntry struct {
+ Name string
+ Timestamp time.Time
+}
+
+// InitializeCacheEviction sets up periodic eviction of expired cache entries.
+func InitializeCacheEviction() {
+ go func() {
+ ticker := time.NewTicker(CacheEntryDuration)
+ defer ticker.Stop()
+
+ for {
+ <-ticker.C // This blocks until the ticker sends a value
+ log.Info().Msg("Evicting Expired API Key Cache Entries")
+ evictExpiredEntries()
+ }
+ }()
+}
+
+// EvictExpiredEntries goes through the cache and evicts expired entries.
+func evictExpiredEntries() {
+ now := time.Now()
+ apiKeyCache.Range(func(key, value interface{}) bool {
+ if entry, ok := value.(cacheEntry); ok {
+ if now.Sub(entry.Timestamp) >= CacheEntryDuration {
+ apiKeyCache.Delete(key)
+ }
+ }
+ return true
+ })
+}
+
+// AuthenticateRequest checks the provided API key against the known keys.
+func AuthenticateRequest(apiKey string) (string, error) {
+ /// Attempt to retrieve API Key from the cache.
+ if val, ok := apiKeyCache.Load(apiKey); ok {
+ entry := val.(cacheEntry)
+ if time.Since(entry.Timestamp) < CacheEntryDuration {
+ return entry.Name, nil
+ }
+ }
+
+ // If the key is not in the cache or the cache has expired, call the db to check the API key.
+ name, err := db.CheckAPIKey(apiKey)
+ if err != nil {
+ if err == sql.ErrNoRows {
+ log.Warn().Msg("Authorization Failed for an API Key")
+ return "", fmt.Errorf("AUTHFAILED")
+ }
+ return "", err
+ }
+
+ // The API key has been successfully authenticated, so cache it.
+ apiKeyCache.Store(apiKey, cacheEntry{Name: name, Timestamp: time.Now()})
+
+ return name, nil
+}
diff --git a/src/ingester/config/config.go b/src/ingester/config/config.go
new file mode 100644
index 00000000..07a7cb9a
--- /dev/null
+++ b/src/ingester/config/config.go
@@ -0,0 +1,130 @@
+package config
+
+import (
+ "fmt"
+ "os"
+ "strconv"
+
+ "github.com/rs/zerolog/log"
+ "gopkg.in/yaml.v2"
+)
+
+type Configuration struct {
+ IngesterPort string `yaml:"ingesterPort"`
+ PricingInfo struct {
+ LocalFile struct {
+ Path string `yaml:"path"`
+ } `yaml:"localFile"`
+ URL string `yaml:"url"`
+ } `yaml:"pricingInfo"`
+ DBConfig struct {
+ DBName string `yaml:"name"`
+ DBUser string `yaml:"username"`
+ DBPassword string `yaml:"password"`
+ DBHost string `yaml:"host"`
+ DBPort string `yaml:"port"`
+ DBSSLMode string `yaml:"sslMode"`
+ MaxOpenConns int `yaml:"maxOpenConns"`
+ MaxIdleConns int `yaml:"maxIdleConns"`
+ DataTableName string `yaml:"dataTable"`
+ APIKeyTableName string `yaml:"apiKeyTable"`
+ } `yaml:"dbConfig"`
+ ObservabilityPlatform struct {
+ Enabled bool `yaml:"enabled"`
+ GrafanaCloud struct {
+ PromURL string `yaml:"promUrl"`
+ PromUsername string `yaml:"promUsername"`
+ LokiURL string `yaml:"lokiUrl"`
+ LokiUsername string `yaml:"lokiUsername"`
+ AccessToken string `yaml:"accessToken"`
+ } `yaml:"grafanaCloud"`
+ NewRelic struct {
+ Key string `yaml:"key"`
+ MetricsURL string `yaml:"metricsUrl"`
+ LogsURL string `yaml:"logsUrl"`
+ } `yaml:"newRelic"`
+ DataDog struct {
+ MetricsURL string `yaml:"metricsUrl"`
+ LogsURL string `yaml:"logsUrl"`
+ APIKey string `yaml:"apiKey"`
+ } `yaml:"datadog"`
+ } `yaml:"observabilityPlatform"`
+}
+
+func validateConfig(cfg *Configuration) error {
+ if _, err := strconv.Atoi(cfg.IngesterPort); err != nil {
+ return fmt.Errorf("Ingester Port is not defined")
+ }
+
+ // Check if at least one PricingInfo configuration is set.
+ if cfg.PricingInfo.LocalFile.Path == "" && cfg.PricingInfo.URL == "" {
+ return fmt.Errorf("PricingInfo configuration is not defined")
+ }
+
+ // Check if both PricingInfo configurations are set.
+ if cfg.PricingInfo.LocalFile.Path != "" && cfg.PricingInfo.URL != "" {
+ return fmt.Errorf("Both LocalFile and URL configurations are defined in PricingInfo; only one is allowed")
+ }
+
+ if cfg.DBConfig.DBPassword == "" {
+ log.Info().Msg("'dbConfig.password' is not defined, trying to read from environment variable 'DB_PASSWORD'")
+ cfg.DBConfig.DBPassword = os.Getenv("DB_PASSWORD")
+ if cfg.DBConfig.DBPassword == "" {
+ return fmt.Errorf("'DB_PASSWORD' environment variable is not set and 'dbConfig.password' is not defined in configuration")
+ }
+ log.Info().Msg("dbConfig.password is now set")
+ }
+
+ if cfg.DBConfig.DBUser == "" {
+ log.Info().Msg("'dbConfig.username' is not defined, trying to read from environment variable 'DB_USERNAME")
+ cfg.DBConfig.DBUser = os.Getenv("DB_USERNAME")
+ if cfg.DBConfig.DBUser == "" {
+ return fmt.Errorf("'DB_USERNAME' environment variable is not set and 'dbConfig.username' is not defined in configuration")
+ }
+ log.Info().Msg("dbConfig.username is now set")
+ }
+
+ if cfg.ObservabilityPlatform.Enabled {
+ definedConfigs := 0
+
+ if cfg.ObservabilityPlatform.GrafanaCloud.PromURL != "" {
+ definedConfigs++
+ }
+ if cfg.ObservabilityPlatform.NewRelic.Key != "" {
+ definedConfigs++
+ }
+ if cfg.ObservabilityPlatform.DataDog.APIKey != "" {
+ definedConfigs++
+ }
+
+ if definedConfigs > 1 {
+ return fmt.Errorf("Only one observability platform configuration (GrafanaCloud, NewRelic, or DataDog) can be enabled at a time")
+ }
+
+ if definedConfigs == 0 {
+ return fmt.Errorf("Observability platform is enabled, but no observability configurations are defined")
+ }
+ }
+
+ return nil
+}
+
+func LoadConfiguration(configPath string) (*Configuration, error) {
+ data, err := os.ReadFile(configPath)
+ if err != nil {
+ return nil, err
+ }
+
+ var cfg Configuration
+ err = yaml.Unmarshal(data, &cfg)
+ if err != nil {
+ return nil, err
+ }
+
+ // Validate the loaded configuration
+ if err := validateConfig(&cfg); err != nil {
+ return nil, err
+ }
+
+ return &cfg, nil
+}
diff --git a/src/ingester/cost/cost.go b/src/ingester/cost/cost.go
new file mode 100644
index 00000000..4539850e
--- /dev/null
+++ b/src/ingester/cost/cost.go
@@ -0,0 +1,163 @@
+package cost
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+)
+
+var Pricing PricingModel
+
+// PricingModel is the pricing information for the different models and features.
+type PricingModel struct {
+ Embeddings map[string]float64 `json:"embeddings"`
+ Images map[string]map[string]map[string]float64 `json:"images"`
+ Audio map[string]float64 `json:"audio"`
+ Chat map[string]struct {
+ PromptPrice float64 `json:"promptPrice"`
+ CompletionPrice float64 `json:"completionPrice"`
+ } `json:"chat"`
+}
+
+// validatePricingData validates the pricing data for the different models and features.
+func validatePricingData(pricingModel PricingModel) error {
+ // Example validation for Embeddings pricing
+ if len(pricingModel.Embeddings) == 0 {
+ return fmt.Errorf("Embeddings pricing data is not defined")
+ }
+
+ // Example validation for Audio pricing
+ if len(pricingModel.Audio) == 0 {
+ return fmt.Errorf("Audio pricing data is not defined")
+ }
+
+ // Validate the Images pricing, which has nested maps
+ for model, qualityMap := range pricingModel.Images {
+ if len(qualityMap) == 0 {
+ return fmt.Errorf("image pricing data for model '%s' is not defined in the JSON File", model)
+ }
+ for quality, sizeMap := range qualityMap {
+ if len(sizeMap) == 0 {
+ return fmt.Errorf("image pricing data for model '%s', quality '%s' is not defined in the JSON File", model, quality)
+ }
+ }
+ }
+
+ // Validate the Chat pricing
+ for model, chatPricing := range pricingModel.Chat {
+ if chatPricing.PromptPrice == 0 {
+ return fmt.Errorf("Prompt Tokens pricing data for model '%s' is not defined in the JSON File", model)
+ } else if chatPricing.CompletionPrice == 0 {
+ return fmt.Errorf("Completion Tokens pricing data for model '%s' is not defined in the JSON File", model)
+ }
+ }
+
+ return nil
+}
+
+// fetchJSONFromURL is a new function for fetching JSON content from a URL.
+func fetchJSONFromURL(url string) ([]byte, error) {
+ maxRetries := 5
+ var resp *http.Response
+ var err error
+ for i := 0; i < maxRetries; i++ {
+ resp, err = http.Get(url)
+ if err == nil {
+ break
+ }
+ }
+
+ if err != nil {
+ return nil, fmt.Errorf("Failed to make request to URL %s", url)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Failed to fetch JSON content from URL %s", url)
+ }
+
+ return io.ReadAll(resp.Body)
+}
+
+// fetchJSONFromFile is a new function for fetching JSON content from a URL.
+func fetchJSONFromFile(path string) ([]byte, error) {
+ content, err := os.ReadFile(path)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to fetch JSON content from file '%s'", path)
+ }
+
+ return content, nil
+}
+
+// LoadPricing loads the pricing information from the given file.
+func LoadPricing(path, url string) error {
+ var content []byte
+ var err error
+
+ switch {
+ case path != "":
+ content, err = fetchJSONFromFile(path)
+ case url != "":
+ content, err = fetchJSONFromURL(url)
+ }
+ if err != nil {
+ return err
+ }
+
+ if err = json.Unmarshal(content, &Pricing); err != nil {
+ return fmt.Errorf("Failed to unmarshal costing JSON: %w", err)
+ }
+
+ if err = validatePricingData(Pricing); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// calculateEmbeddingsCost calculates the cost for embeddings based on the model and prompt tokens.
+func CalculateEmbeddingsCost(promptTokens float64, model string) (float64, error) {
+ price, ok := Pricing.Embeddings[model]
+ if !ok {
+ return 0, nil
+ }
+ return (promptTokens / 1000) * price, nil
+}
+
+// calculateImageCost calculates the cost for images based on the model, image size, and quality.
+func CalculateImageCost(model, imageSize, quality string) (float64, error) {
+ models, ok := Pricing.Images[model]
+ if !ok {
+ return 0, nil
+ }
+ qualities, ok := models[quality]
+ if !ok {
+ return 0, nil
+ }
+ price, ok := qualities[imageSize]
+ if !ok {
+ return 0, nil
+ }
+
+ return price, nil
+}
+
+// calculateChatCost calculates the cost for chat based on the model, prompt tokens, and completion tokens.
+func CalculateChatCost(promptTokens, completionTokens float64, model string) (float64, error) {
+ chatModel, ok := Pricing.Chat[model]
+ if !ok {
+ return 0, nil
+ }
+ return ((promptTokens / 1000) * chatModel.PromptPrice) + ((completionTokens / 1000) * chatModel.CompletionPrice), nil
+}
+
+// CalculateAudioCost calculates the cost for Audio based on the model, and prompt.
+func CalculateAudioCost(prompt string, model string) (float64, error) {
+ price, ok := Pricing.Audio[model]
+ if !ok {
+ return 0, nil
+ }
+ return ((float64(len(prompt)) / 1000) * price), nil
+}
diff --git a/src/ingester/db/db.go b/src/ingester/db/db.go
new file mode 100644
index 00000000..13a8f9d2
--- /dev/null
+++ b/src/ingester/db/db.go
@@ -0,0 +1,437 @@
+package db
+
+import (
+ "crypto/rand"
+ "database/sql"
+ "encoding/hex"
+ "fmt"
+ "ingester/config"
+ "ingester/cost"
+ "ingester/obsPlatform"
+ "net/http"
+ "sync"
+
+ _ "github.com/lib/pq"
+ "github.com/pkoukk/tiktoken-go"
+ "github.com/rs/zerolog/log"
+)
+
+var (
+ once sync.Once // once is used to ensure that the database is initialized only once
+ db *sql.DB // db holds the database connection
+ dbConfig DatabaseConfig // dbConfig holds the database configuration
+
+ // validFields represent the fields that are expected in the incoming data.
+ validFields = []string{
+ "name",
+ "environment",
+ "endpoint",
+ "sourceLanguage",
+ "applicationName",
+ "completionTokens",
+ "promptTokens",
+ "totalTokens",
+ "finishReason",
+ "requestDuration",
+ "usageCost",
+ "model",
+ "prompt",
+ "response",
+ "imageSize",
+ "revisedPrompt",
+ "image",
+ "audioVoice",
+ "finetuneJobId",
+ "finetuneJobStatus",
+ }
+)
+
+// DBConfig holds the database configuration
+type DatabaseConfig struct {
+ DBName string
+ User string
+ Password string
+ Host string
+ Port string
+ SSLMode string
+ MaxIdleConns int
+ MaxOpenConns int
+ DataTableName string
+ ApiKeyTableName string
+}
+
+// PingDB attempts to ping the database to check if it's alive.
+func PingDB() error {
+ return db.Ping()
+}
+
+// GenerateSecureRandomKey should generate a secure random string to be used as an API key.
+func generateSecureRandomKey() (string, error) {
+ randomPartLength := 40 / 2 // Each byte becomes two hex characters, so we need half as many bytes.
+
+ randomBytes := make([]byte, randomPartLength)
+ _, err := rand.Read(randomBytes)
+ if err != nil {
+ log.Error().Err(err).Msg("Error generating random bytes")
+ // In the case of an error, return it as we cannot generate a key.
+ return "", err
+ }
+
+ // Encode the random bytes as a hex string and prefix with 'dk'.
+ randomHexString := hex.EncodeToString(randomBytes)
+ apiKey := "dk" + randomHexString
+
+ return apiKey, nil
+}
+
+// getCreateAPIKeysTableSQL returns the SQL query to create the API keys table.
+func getCreateAPIKeysTableSQL(tableName string) string {
+ return fmt.Sprintf(`
+ CREATE TABLE IF NOT EXISTS %s (
+ id SERIAL PRIMARY KEY,
+ api_key VARCHAR(255) NOT NULL UNIQUE,
+ name VARCHAR(50) NOT NULL
+ );`, tableName)
+}
+
+// getCreateDataTableSQL returns the SQL query to create the data table.
+func getCreateDataTableSQL(tableName string) string {
+ return fmt.Sprintf(`
+ CREATE TABLE IF NOT EXISTS %s (
+ time TIMESTAMPTZ NOT NULL,
+ name VARCHAR(10) NOT NULL,
+ environment VARCHAR(50) NOT NULL,
+ endpoint VARCHAR(50) NOT NULL,
+ sourceLanguage VARCHAR(50) NOT NULL,
+ applicationName VARCHAR(50) NOT NULL,
+ completionTokens INTEGER,
+ promptTokens INTEGER,
+ totalTokens INTEGER,
+ finishReason VARCHAR(50),
+ requestDuration DOUBLE PRECISION,
+ usageCost DOUBLE PRECISION,
+ model VARCHAR(50),
+ prompt TEXT,
+ response TEXT,
+ imageSize TEXT,
+ revisedPrompt TEXT,
+ image TEXT,
+ audioVoice TEXT,
+ finetuneJobId TEXT,
+ finetuneJobStatus TEXT
+ );`, tableName)
+}
+
+// tableExists checks if a table exists in the database.
+func tableExists(db *sql.DB, tableName string) (bool, error) {
+ query := `
+ SELECT EXISTS (
+ SELECT 1
+ FROM information_schema.tables
+ WHERE table_schema = 'public'
+ AND lower(table_name) = lower($1)
+ )
+ `
+
+ var exists bool
+ err := db.QueryRow(query, tableName).Scan(&exists)
+ if err != nil {
+ return false, err
+ }
+
+ return exists, nil
+}
+
+// createTable creates a table in the database if it doesn't exist.
+func createTable(db *sql.DB, tableName string) error {
+ var createTableSQL string
+ if tableName == dbConfig.ApiKeyTableName {
+ createTableSQL = getCreateAPIKeysTableSQL(tableName)
+ } else if tableName == dbConfig.DataTableName {
+ createTableSQL = getCreateDataTableSQL(tableName)
+ }
+
+ exists, err := tableExists(db, tableName)
+ if err != nil {
+ return fmt.Errorf("Error checking table '%s' existence: %w", tableName, err)
+ }
+ if exists == false {
+ _, err := db.Exec(createTableSQL)
+ if err != nil {
+ return fmt.Errorf("Error creating table %s: %w", tableName, err)
+ }
+ log.Info().Msgf("Table '%s' created in the database", tableName)
+
+ if tableName == dbConfig.ApiKeyTableName {
+ createIndexSQL := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_api_key ON %s (api_key);", tableName)
+ _, err = db.Exec(createIndexSQL)
+ if err != nil {
+ return fmt.Errorf("Error creating index on 'api_key' column: %w", err)
+ }
+ log.Info().Msgf("Index on 'api_key' column checked/created in table '%s'", tableName)
+ }
+
+ // If the table to create is the data table, convert it into a hypertable
+ if tableName == dbConfig.DataTableName {
+ _, err := db.Exec("SELECT create_hypertable($1, 'time')", tableName)
+ if err != nil {
+ return fmt.Errorf("Error creating hypertable: %w", err)
+ }
+ log.Info().Msgf("Table '%s' converted to a Hypertable", tableName)
+ }
+ } else {
+ log.Info().Msgf("Table '%s' already exists in the database", tableName)
+ }
+ return nil
+}
+
+// initializeDB initializes connection to the database.
+func initializeDB() error {
+ var dbErr error
+ once.Do(func() {
+ connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
+ dbConfig.Host, dbConfig.Port, dbConfig.User, dbConfig.Password, dbConfig.DBName, dbConfig.SSLMode)
+
+ if db, dbErr = sql.Open("postgres", connStr); dbErr != nil {
+ log.Error().Err(dbErr).Msg("Error connecting to the database")
+ return
+ }
+
+ dbErr = db.Ping()
+ if dbErr != nil {
+ return
+ }
+
+ db.SetMaxOpenConns(dbConfig.MaxOpenConns)
+ db.SetMaxIdleConns(dbConfig.MaxIdleConns)
+ })
+ return dbErr
+}
+
+func getTokens(text, model string) int {
+ tkm, err := tiktoken.EncodingForModel(model)
+ if err != nil {
+ tkm, _ = tiktoken.GetEncoding("cl100k_base")
+ }
+ token := tkm.Encode(text, nil, nil)
+ return len(token)
+}
+
+// insertDataToDB inserts data into the database.
+func insertDataToDB(data map[string]interface{}) (string, int) {
+ // Calculate usage cost based on the endpoint type
+ if data["endpoint"] == "openai.embeddings" || data["endpoint"] == "cohere.embed" {
+ data["usageCost"], _ = cost.CalculateEmbeddingsCost(data["promptTokens"].(float64), data["model"].(string))
+ } else if data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions" || data["endpoint"] == "cohere.chat" || data["endpoint"] == "cohere.summarize" || data["endpoint"] == "cohere.generate" {
+ if data["completionTokens"] != nil && data["promptTokens"] != nil {
+ data["usageCost"], _ = cost.CalculateChatCost(data["promptTokens"].(float64), data["completionTokens"].(float64), data["model"].(string))
+ } else if (data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions") && data["prompt"] != nil && data["response"] != nil {
+ data["promptTokens"] = getTokens(data["prompt"].(string), data["model"].(string))
+ data["completionTokens"] = getTokens(data["response"].(string), data["model"].(string))
+ data["totalTokens"] = data["promptTokens"].(int) + data["completionTokens"].(int)
+ data["usageCost"], _ = cost.CalculateChatCost(float64(data["promptTokens"].(int)), float64(data["completionTokens"].(int)), data["model"].(string))
+ }
+ } else if data["endpoint"] == "openai.images.create" || data["endpoint"] == "openai.images.create.variations" {
+ data["usageCost"], _ = cost.CalculateImageCost(data["model"].(string), data["imageSize"].(string), data["imageQuality"].(string))
+ } else if data["endpoint"] == "openai.audio.speech.create" {
+ data["usageCost"], _ = cost.CalculateAudioCost(data["prompt"].(string), data["model"].(string))
+ }
+
+ // Fill missing fields with nil
+ for _, field := range validFields {
+ if _, exists := data[field]; !exists {
+ data[field] = nil
+ }
+ }
+
+ go obsPlatform.SendToPlatform(data)
+
+ // Define the SQL query for data insertion
+ query := fmt.Sprintf("INSERT INTO %s (time, name, environment, endpoint, sourceLanguage, applicationName, completionTokens, promptTokens, totalTokens, finishReason, requestDuration, usageCost, model, prompt, response, imageSize, revisedPrompt, image, audioVoice, finetuneJobId, finetuneJobStatus) VALUES (NOW(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)", dbConfig.DataTableName)
+
+ // Execute the SQL query
+ _, err := db.Exec(query,
+ data["name"],
+ data["environment"],
+ data["endpoint"],
+ data["sourceLanguage"],
+ data["applicationName"],
+ data["completionTokens"],
+ data["promptTokens"],
+ data["totalTokens"],
+ data["finishReason"],
+ data["requestDuration"],
+ data["usageCost"],
+ data["model"],
+ data["prompt"],
+ data["response"],
+ data["imageSize"],
+ data["revisedPrompt"],
+ data["image"],
+ data["audioVoice"],
+ data["finetuneJobId"],
+ data["finetuneJobStatus"],
+ )
+ if err != nil {
+ log.Error().Err(err).Msg("Error Inserting data into the database")
+ // Update the response message and status code for error
+ return "Internal Server Error", http.StatusInternalServerError
+ }
+
+ return "Data insertion completed", http.StatusCreated
+}
+
+// Init initializes the database connection and creates the required tables.
+func Init(cfg config.Configuration) error {
+
+ // Initialize the database configuration
+ dbConfig = DatabaseConfig{
+ DBName: cfg.DBConfig.DBName,
+ User: cfg.DBConfig.DBUser,
+ Password: cfg.DBConfig.DBPassword,
+ Host: cfg.DBConfig.DBHost,
+ Port: cfg.DBConfig.DBPort,
+ SSLMode: cfg.DBConfig.DBSSLMode,
+ MaxIdleConns: cfg.DBConfig.MaxIdleConns,
+ MaxOpenConns: cfg.DBConfig.MaxOpenConns,
+ DataTableName: cfg.DBConfig.DataTableName,
+ ApiKeyTableName: cfg.DBConfig.APIKeyTableName,
+ }
+
+ err := initializeDB()
+ if err != nil {
+ log.Error().Err(err).Msg("Error initializing database")
+ return fmt.Errorf("Could not initialize connection to the database: %w", err)
+ }
+
+ // Create the DATA and API keys table if it doesn't exist.
+ log.Info().Msgf("Creating '%s' and '%s' tables in the database if they don't exist", dbConfig.ApiKeyTableName, dbConfig.DataTableName)
+
+ err = createTable(db, dbConfig.ApiKeyTableName)
+ if err != nil {
+ log.Error().Err(err).Msgf("Error creating table %s", dbConfig.ApiKeyTableName)
+ return err
+ }
+
+ err = createTable(db, dbConfig.DataTableName)
+ if err != nil {
+ log.Error().Err(err).Msgf("Error creating table %s", dbConfig.ApiKeyTableName)
+ return err
+ }
+ return nil
+}
+
+// PerformDatabaseInsertion performs the database insertion synchronously.
+func PerformDatabaseInsertion(data map[string]interface{}) (string, int) {
+ // Call insertDataToDB directly instead of starting a new goroutine.
+ responseMessage, statusCode := insertDataToDB(data)
+
+ // The operation is now synchronous. Once insertDataToDB returns, the result is ready to use.
+ return responseMessage, statusCode
+}
+
+// CheckAPIKey retrieves the name associated with the given API key from the database.
+func CheckAPIKey(apiKey string) (string, error) {
+ var name string
+
+ query := fmt.Sprintf("SELECT name FROM %s WHERE api_key = $1", dbConfig.ApiKeyTableName)
+ err := db.QueryRow(query, apiKey).Scan(&name)
+ if err != nil {
+ if err == sql.ErrNoRows {
+ return "", err
+ }
+ return "", err
+ }
+ return name, nil
+}
+
+// GenerateAPIKey generates a new API key for a given name and stores it in the database.
+func GenerateAPIKey(existingAPIKey, name string) (string, error) {
+ // If there are any existing API keys, authenticate the provided API key before proceeding
+ var count int
+ countQuery := fmt.Sprintf("SELECT COUNT(*) FROM %s", dbConfig.ApiKeyTableName)
+ err := db.QueryRow(countQuery).Scan(&count)
+ if err != nil {
+ log.Error().Err(err).Msg("Error checking API key table")
+ return "", fmt.Errorf("Failed to check API key table: %v", err)
+ }
+
+ // Only perform the check if the count is greater than zero
+ if count > 0 {
+ // Attempt to retrieve any existing key for the given name.
+ _, err = GetAPIKeyForName(existingAPIKey, name)
+ if err == nil {
+ log.Warn().Msgf("Error creating new API Key as a key with the name '%s' already exists", name)
+ return "", fmt.Errorf("KEYEXISTS")
+ } else if err.Error() == "AUTHFAILED" {
+ return "", err
+ }
+ }
+
+ // No existing key found, proceed to generate a new API key
+ log.Info().Msgf("Creating a new API Key with the name: %s", name)
+ newAPIKey, _ := generateSecureRandomKey()
+
+ // Insert the new API key into the database
+ insertQuery := fmt.Sprintf("INSERT INTO %s (api_key, name) VALUES ($1, $2)", dbConfig.ApiKeyTableName)
+ _, err = db.Exec(insertQuery, newAPIKey, name)
+ if err != nil {
+ log.Error().Err(err).Msg("Error inserting the new API key in the database")
+ return "", err
+ }
+ log.Info().Msgf("API Key with the name '%s' created successfully", name)
+ return newAPIKey, nil
+}
+
+// GetAPIKeyForName retrieves an API key for a given name from the database.
+func GetAPIKeyForName(existingAPIKey, name string) (string, error) {
+
+ // Autheticate the provided API key before proceeding
+ _, err := CheckAPIKey(existingAPIKey)
+ if err != nil {
+ log.Warn().Msg("Authorization Failed for an API Key")
+ return "", fmt.Errorf("AUTHFAILED")
+ }
+
+ // Retrieve the API key for the given name
+ var apiKey string
+ query := fmt.Sprintf("SELECT api_key FROM %s WHERE name = $1", dbConfig.ApiKeyTableName)
+ err = db.QueryRow(query, name).Scan(&apiKey)
+ if err != nil {
+ if err == sql.ErrNoRows {
+ log.Warn().Msgf("API Key with the name '%s' currently not found in the database", name)
+ return "", fmt.Errorf("NOTFOUND")
+ }
+ log.Warn().Err(err).Msgf("Error retrieving API key for the name '%s'", name)
+ return "", err
+ }
+
+ return apiKey, nil
+}
+
+// DeleteAPIKey deletes an API key for a given name from the database.
+func DeleteAPIKey(existingAPIKey, name string) error {
+
+ // Autheticate the provided API key before proceeding and check if the API key exists
+ apiKey, err := GetAPIKeyForName(existingAPIKey, name)
+ if err != nil {
+ if err.Error() == "AUTHFAILED" {
+ return err
+ }
+ if err.Error() == "NOTFOUND" {
+ return err
+ }
+ return err
+ }
+
+ // Delete the API key from the database
+ log.Info().Msgf("Deleting API Key with the name '%s' from the database", name)
+ query := fmt.Sprintf("DELETE FROM %s WHERE api_key = $1", dbConfig.ApiKeyTableName)
+ _, err = db.Exec(query, apiKey)
+ if err != nil {
+ log.Error().Err(err).Msg("Error deleting API key")
+ return err
+ }
+ log.Info().Msgf("API Key with the name '%s' deleted successfully", name)
+ return nil
+}
diff --git a/src/ingester/go.mod b/src/ingester/go.mod
new file mode 100644
index 00000000..16b21074
--- /dev/null
+++ b/src/ingester/go.mod
@@ -0,0 +1,20 @@
+module ingester
+
+go 1.21.5
+
+require (
+ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be
+ github.com/gorilla/mux v1.8.1
+ github.com/lib/pq v1.10.9
+ github.com/pkoukk/tiktoken-go v0.1.6
+ github.com/rs/zerolog v1.31.0
+ gopkg.in/yaml.v2 v2.4.0
+)
+
+require (
+ github.com/dlclark/regexp2 v1.10.0 // indirect
+ github.com/google/uuid v1.3.0 // indirect
+ github.com/mattn/go-colorable v0.1.13 // indirect
+ github.com/mattn/go-isatty v0.0.19 // indirect
+ golang.org/x/sys v0.12.0 // indirect
+)
diff --git a/src/ingester/go.sum b/src/ingester/go.sum
new file mode 100644
index 00000000..37120922
--- /dev/null
+++ b/src/ingester/go.sum
@@ -0,0 +1,39 @@
+github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=
+github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=
+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=
+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/src/ingester/main.go b/src/ingester/main.go
new file mode 100644
index 00000000..fb7acbde
--- /dev/null
+++ b/src/ingester/main.go
@@ -0,0 +1,123 @@
+package main
+
+import (
+ "context"
+ "flag"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+
+ "ingester/api"
+ "ingester/auth"
+ "ingester/config"
+ "ingester/cost"
+ "ingester/db"
+ "ingester/obsPlatform"
+
+ "github.com/common-nighthawk/go-figure"
+ "github.com/gorilla/mux"
+ "github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
+)
+
+func waitForShutdown(server *http.Server) {
+ quit := make(chan os.Signal, 1)
+ signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
+
+ // Wait for an interrupt
+ sig := <-quit
+ log.Info().Msgf("Caught sig: %+v", sig)
+ log.Info().Msg("Starting to shutdown server")
+
+ // Initialize the context with a timeout to ensure the app can make a graceful exit
+ // or abort if it takes too long
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ if err := server.Shutdown(ctx); err != nil {
+ log.Error().Err(err).Msg("Server shutdown failed")
+ } else {
+ log.Info().Msg("Server gracefully shutdown")
+ }
+}
+
+// main is the entrypoint for the Doku Ingester service. It sets up logging,
+// initializes the database and observability platforms, starts the HTTP server,
+// and handles graceful shutdown.
+func main() {
+ figure.NewColorFigure("DOKU Ingester", "", "yellow", true).Print()
+ // Configure global settings for the zerolog logger
+ zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
+ log.Info().Msg("Starting Doku Ingester")
+
+ // Use flag package to parse the configuration file
+ configFilePath := flag.String("config", "./config.yml", "Path to the Doku Ingester config file")
+ flag.Parse()
+
+ // Load the configuration
+ cfg, err := config.LoadConfiguration(*configFilePath)
+ if err != nil {
+ log.Fatal().Err(err).Msg("Failed to load configuration file")
+ }
+
+ // Load pricing information, either from a local file or from a URL
+ if cfg.PricingInfo.URL != "" {
+ log.Info().Msgf("Initializing LLM Pricing Information from URL '%s'", cfg.PricingInfo.URL)
+ err = cost.LoadPricing("", cfg.PricingInfo.URL)
+ } else if cfg.PricingInfo.LocalFile.Path != "" {
+ log.Info().Msgf("Initializing LLM Pricing Information from local file '%s", cfg.PricingInfo.LocalFile.Path)
+ err = cost.LoadPricing(cfg.PricingInfo.LocalFile.Path, "")
+ }
+ if err != nil {
+ log.Fatal().Err(err).Msg("Failed to load LLM pricing information")
+ }
+ log.Info().Msg("Successfully initialized LLM pricing information")
+
+ // Initialize the backend database connection with loaded configuration
+ log.Info().Msg("Initializing connection to the backend database")
+ err = db.Init(*cfg)
+ if err != nil {
+ log.Fatal().Msg("Unable to initialize connection to the backend database")
+ }
+ log.Info().Msg("Successfully initialized connection to the backend database")
+
+ // Initialize observability platform if configured
+ if cfg.ObservabilityPlatform.Enabled == true {
+ log.Info().Msg("Initializing for your Observability Platform")
+ err := obsPlatform.Init(*cfg)
+ if err != nil {
+ log.Fatal().Msg("Exiting due to error in initializing for your Observability Platform")
+ }
+ log.Info().Msgf("Setup complete for sending data to %s", obsPlatform.ObservabilityPlatform)
+ }
+
+ // Cache eviction setup for the authentication process
+ auth.InitializeCacheEviction()
+
+ // Initialize the HTTP server routing
+ r := mux.NewRouter()
+ r.HandleFunc("/api/push", api.DataHandler).Methods("POST")
+ r.HandleFunc("/api/keys", api.APIKeyHandler).Methods("GET", "POST", "DELETE")
+ r.HandleFunc("/", api.BaseEndpoint).Methods("GET")
+
+ // Define and start the HTTP server
+ server := &http.Server{
+ Addr: ":" + cfg.IngesterPort,
+ Handler: r,
+ ReadTimeout: 10 * time.Second,
+ WriteTimeout: 10 * time.Second,
+ IdleTimeout: 30 * time.Second,
+ }
+
+ // Starts the HTTP server in a goroutine and logs any error upon starting.
+ go func() {
+ log.Info().Msg("Server listening on port " + cfg.IngesterPort)
+ if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+ log.Fatal().Err(err).Msg("Could not listen on port " + cfg.IngesterPort)
+ }
+ }()
+
+ waitForShutdown(server)
+}
diff --git a/src/ingester/obsPlatform/datadog.go b/src/ingester/obsPlatform/datadog.go
new file mode 100644
index 00000000..c1b0e709
--- /dev/null
+++ b/src/ingester/obsPlatform/datadog.go
@@ -0,0 +1,335 @@
+package obsPlatform
+
+import (
+ "bytes"
+ "fmt"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/rs/zerolog/log"
+)
+
+func configureDataDogData(data map[string]interface{}) {
+ currentTime := time.Now().Unix()
+
+ if data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions" || data["endpoint"] == "cohere.generate" || data["endpoint"] == "cohere.chat" || data["endpoint"] == "cohere.summarize" || data["endpoint"] == "anthropic.completions" {
+ if data["finishReason"] == nil {
+ data["finishReason"] = "null"
+ }
+ // Create individual metric strings
+ metricStrings := []string{
+ fmt.Sprintf(`{
+ "metric": "doku.llm.completion.tokens",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]
+ }`, currentTime, data["completionTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.prompt.tokens",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]
+ }`, currentTime, data["promptTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.total.tokens",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]
+ }`, currentTime, data["totalTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.request.duration",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]
+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.usage.cost",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]
+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ }
+
+ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))
+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")
+ }
+
+ logEntries := []string{
+ fmt.Sprintf(`{
+ "ddsource": "doku",
+ "message": "%s",
+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",
+ "hostname": "doku-ingester",
+ "service": "%v"
+ }`, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),
+ fmt.Sprintf(`{
+ "ddsource": "doku",
+ "message": "%s",
+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:response",
+ "hostname": "doku-ingester",
+ "service": "%v"
+ }`, normalizeString(data["response"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),
+ }
+
+ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))
+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")
+ }
+ } else if data["endpoint"] == "openai.embeddings" || data["endpoint"] == "cohere.embed" {
+ if data["endpoint"] == "openai.embeddings" {
+
+ metricStrings := []string{
+ fmt.Sprintf(`{
+ "metric": "doku.llm.prompt.tokens",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]
+ }`, currentTime, data["promptTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.total.tokens",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]
+ }`, currentTime, data["totalTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.request.duration",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]
+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.usage.cost",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]
+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ }
+
+ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))
+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")
+ }
+
+ logEntries := []string{
+ fmt.Sprintf(`{
+ "ddsource": "doku",
+ "message": "%s",
+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",
+ "hostname": "doku-ingester",
+ "service": "%v"
+ }`, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),
+ }
+
+ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))
+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")
+ }
+ } else {
+ metricStrings := []string{
+ fmt.Sprintf(`{
+ "metric": "doku.llm.prompt.tokens",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]
+ }`, currentTime, data["promptTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.request.duration",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]
+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.usage.cost",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]
+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ }
+
+ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))
+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")
+ }
+
+ logEntries := []string{
+ fmt.Sprintf(`{
+ "ddsource": "doku",
+ "message": "%s",
+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",
+ "hostname": "doku-ingester",
+ "service": "%v"
+ }`, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),
+ }
+
+ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))
+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")
+ }
+ }
+ } else if data["endpoint"] == "openai.fine_tuning" {
+ metricStrings := []string{
+ fmt.Sprintf(`{
+ "metric": "doku.llm.request.duration",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finetuneJobId:%v"]
+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finetuneJobId"]),
+ }
+
+ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))
+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")
+ }
+ } else if data["endpoint"] == "openai.images.create" || data["endpoint"] == "openai.images.create.variations" {
+ metricStrings := []string{
+ fmt.Sprintf(`{
+ "metric": "doku.llm.request.duration",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "imageSize:%v","imageQuality:%v"]
+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.usage.cost",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "imageSize:%v","imageQuality:%v"]
+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"]),
+ }
+
+ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))
+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")
+ }
+
+ var logEntries []string
+ // Check the condition for the prompt
+ if data["endpoint"] != "openai.images.create.variations" {
+ var promptMessage string
+ if data["model"] == "dall-e-2" {
+ // Assuming data["prompt"] exists and is a string
+ promptMessage = normalizeString(data["prompt"].(string))
+ } else {
+ // Assuming data["revisedPrompt"] exists and is a string
+ promptMessage = normalizeString(data["revisedPrompt"].(string))
+ }
+
+ // Build the prompt log
+ logEntries = append(logEntries, fmt.Sprintf(`{
+ "ddsource": "doku",
+ "message": "%s",
+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",
+ "hostname": "doku-ingester",
+ "service": "%v"
+ }`, promptMessage, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),
+ )
+ }
+
+ logEntries = append(logEntries, fmt.Sprintf(`{
+ "ddsource": "doku",
+ "message": "%s",
+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:image",
+ "hostname": "doku-ingester",
+ "service": "%v"
+ }`, data["image"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),
+ )
+
+ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))
+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")
+ }
+ } else if data["endpoint"] == "openai.audio.speech.create" {
+ metricStrings := []string{
+ fmt.Sprintf(`{
+ "metric": "doku.llm.request.duration",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "audioVoice:%v"]
+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"]),
+ fmt.Sprintf(`{
+ "metric": "doku.llm.usage.cost",
+ "type": 0,
+ "points": [{ "timestamp": %d, "value": %f }],
+ "resources": [{ "name": "doku-ingester", "type": "host" }],
+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "audioVoice:%v"]
+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"]),
+ }
+
+ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))
+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")
+ }
+
+ logEntries := []string{
+ fmt.Sprintf(`{
+ "ddsource": "doku",
+ "message": "%s",
+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",
+ "hostname": "doku-ingester",
+ "service": "%v"
+ }`, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),
+ }
+
+ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))
+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")
+ }
+ }
+}
+
+func sendTelemetryDataDog(telemetryData, headerKey string, url string, requestType string) error {
+ // Create a new request using http
+ req, err := http.NewRequest(requestType, url, bytes.NewBuffer([]byte(telemetryData)))
+ if err != nil {
+ return fmt.Errorf("Error creating request")
+ }
+
+ // Add headers to the request
+ req.Header.Set("Accept", "application/json")
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("DD-API-KEY", headerKey)
+
+ // Send the request via a client
+ httpClient := &http.Client{}
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("Error sending request to %v", url)
+ } else if resp.StatusCode == 404 {
+ return fmt.Errorf("Provided URL %v is not valid", url)
+ } else if resp.StatusCode == 403 {
+ return fmt.Errorf("Provided credentials are not valid")
+ }
+
+ defer resp.Body.Close()
+
+ log.Info().Msgf("Successfully exported data to %v", url)
+ return nil
+}
diff --git a/src/ingester/obsPlatform/newrelic.go b/src/ingester/obsPlatform/newrelic.go
new file mode 100644
index 00000000..2b0499ea
--- /dev/null
+++ b/src/ingester/obsPlatform/newrelic.go
@@ -0,0 +1,416 @@
+package obsPlatform
+
+import (
+ "bytes"
+ "fmt"
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/rs/zerolog/log"
+)
+
+func configureNewRelicData(data map[string]interface{}) {
+ // The current time for the timestamp field.
+ currentTime := strconv.FormatInt(time.Now().Unix(), 10)
+
+ if data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions" || data["endpoint"] == "cohere.generate" || data["endpoint"] == "cohere.chat" || data["endpoint"] == "cohere.summarize" || data["endpoint"] == "anthropic.completions" {
+ if data["finishReason"] == nil {
+ data["finishReason"] = "null"
+ }
+
+ jsonMetrics := []string{
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Completion.Tokens",
+ "type": "gauge",
+ "value": %f,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}
+ }`, data["completionTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Prompt.Tokens",
+ "type": "gauge",
+ "value": %f,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}
+ }`, data["promptTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Total.Tokens",
+ "type": "gauge",
+ "value": %f,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}
+ }`, data["totalTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Request.Duration",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}
+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Usage.Cost",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}
+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),
+ }
+
+ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.
+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))
+
+ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")
+ }
+
+ // Use the values from the provided log lines and adapt them to the desired format.
+ logs := []string{
+ fmt.Sprintf(`{
+ "timestamp": %s,
+ "message": "%s",
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "type": "response"
+ }
+ }`, currentTime, normalizeString(data["response"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]), // Assuming 'response' is present in data and is a string.
+
+ fmt.Sprintf(`{
+ "timestamp": %s,
+ "message": "%s",
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "type": "prompt"
+ }
+ }`, currentTime, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]), // Assuming 'prompt' is present in data and is a string.
+ }
+
+ // Combine the individual log entries into a full JSON payload
+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))
+
+ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")
+ }
+
+ } else if data["endpoint"] == "openai.embeddings" || data["endpoint"] == "cohere.embed" {
+ if data["endpoint"] == "openai.embeddings" {
+ jsonMetrics := []string{
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Prompt.Tokens",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": { "environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}
+ }`, data["promptTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Total.Tokens",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}
+ }`, data["totalTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Request.Duration",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": { "environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}
+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Usage.Cost",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": { "environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}
+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ }
+
+ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.
+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))
+
+ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")
+ }
+
+ // Build log entries with varying labels
+ logs := []string{
+ fmt.Sprintf(`{
+ "timestamp": %s,
+ "message": "%s",
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "type": "prompt"
+ }
+ }`, currentTime, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ }
+ // Combine the individual log entries into a full JSON payload
+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))
+
+ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")
+ }
+ } else {
+ jsonMetrics := []string{
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Prompt.Tokens",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}
+ }`, data["promptTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Request.Duration",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}
+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Usage.Cost",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}
+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ }
+
+ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.
+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))
+
+ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Merics to New Relic")
+ }
+
+ logs := []string{
+ // Assuming 'prompt' exists and is a string in data
+ fmt.Sprintf(`{
+ "timestamp": "%s",
+ "message": "%s",
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "type": "prompt"
+ }
+ }`,
+ currentTime,
+ data["prompt"],
+ data["environment"],
+ data["endpoint"],
+ data["applicationName"],
+ data["sourceLanguage"],
+ data["model"]),
+ }
+
+ // Combine the individual log entries into a full JSON payload
+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))
+
+ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")
+ }
+
+ }
+ } else if data["endpoint"] == "openai.fine_tuning" {
+ jsonMetrics := []string{
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Request.Duration",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finetuneJobId": "%v"}
+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finetuneJobId"]),
+ }
+ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.
+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))
+
+ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")
+ }
+ } else if data["endpoint"] == "openai.images.create" || data["endpoint"] == "openai.images.create.variations" {
+ jsonMetrics := []string{
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Request.Duration",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "imageSize": "%v",
+ "imageQuality": "%v"
+ }
+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"]),
+ fmt.Sprintf(`{
+ "name": "doku.LLM.Usage.Cost",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "imageSize": "%v",
+ "imageQuality": "%v"
+ }
+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"]),
+ }
+
+ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.
+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))
+
+ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")
+ }
+
+ var logs []string
+ // Check the condition for the prompt
+ if data["endpoint"] != "openai.images.create.variations" {
+ var promptMessage string
+ if data["model"] == "dall-e-2" {
+ // Assuming data["prompt"] exists and is a string
+ promptMessage = normalizeString(data["prompt"].(string))
+ } else {
+ // Assuming data["revisedPrompt"] exists and is a string
+ promptMessage = normalizeString(data["revisedPrompt"].(string))
+ }
+
+ // Build the prompt log
+ logs = append(logs, fmt.Sprintf(`{
+ "timestamp": "%s",
+ "message": "%s",
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "type": "prompt"
+ }
+ }`, currentTime, promptMessage, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]))
+ }
+
+ // Build the image log
+ logs = append(logs, fmt.Sprintf(`{
+ "timestamp": "%s",
+ "message": "%s",
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "type": "image"
+ }
+ }`, currentTime, data["image"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]))
+
+ // Combine the individual log entries into a full JSON payload
+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))
+
+ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")
+ }
+
+ } else if data["endpoint"] == "openai.audio.speech.create" {
+ jsonMetrics := []string{
+ fmt.Sprintf(`{
+ "name": "doku_llm.RequestDuration",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "audioVoice": "%v"}
+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"]),
+ fmt.Sprintf(`{
+ "name": "doku_llm.UsageCost",
+ "type": "gauge",
+ "value": %v,
+ "timestamp": %s,
+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "audioVoice": "%v"}
+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"]),
+ }
+
+ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.
+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))
+
+ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")
+ }
+
+ logs := []string{
+ // Assuming 'prompt' exists and is a string in data
+ fmt.Sprintf(`{
+ "timestamp": "%s",
+ "message": "%s",
+ "attributes": {
+ "environment": "%v",
+ "endpoint": "%v",
+ "applicationName": "%v",
+ "source": "%v",
+ "model": "%v",
+ "type": "prompt"
+ }
+ }`, currentTime, data["prompt"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),
+ }
+
+ // Combine the individual log entries into a full JSON payload
+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))
+
+ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")
+ }
+ }
+}
+
+func sendTelemetryNewRelic(telemetryData, authHeader string, headerKey string, url string, requestType string) error {
+
+ req, err := http.NewRequest(requestType, url, bytes.NewBufferString(telemetryData))
+ if err != nil {
+ return fmt.Errorf("Error creating request")
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set(fmt.Sprintf("%s", headerKey), authHeader)
+
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("Error sending request to %v", url)
+ } else if resp.StatusCode == 404 {
+ return fmt.Errorf("Provided URL %v is not valid", url)
+ } else if resp.StatusCode == 403 {
+ return fmt.Errorf("Provided credentials are not valid")
+ }
+
+ defer resp.Body.Close()
+
+ log.Info().Msgf("Successfully exported data to %v", url)
+ return nil
+}
diff --git a/src/ingester/obsPlatform/obsPlatform.go b/src/ingester/obsPlatform/obsPlatform.go
new file mode 100644
index 00000000..6b76074e
--- /dev/null
+++ b/src/ingester/obsPlatform/obsPlatform.go
@@ -0,0 +1,237 @@
+package obsPlatform
+
+import (
+ "bytes"
+ "fmt"
+ "ingester/config"
+ "net/http"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/rs/zerolog/log"
+)
+
+var (
+ ObservabilityPlatform string // ObservabilityPlatform contains the information on the platform in use.
+ httpClient *http.Client // httpClient is the HTTP client used to send data to the Observability Platform.
+ grafanaPromUrl string // grafanaPrometheusUrl is the URL used to send data to Grafana Prometheus.
+ grafanaPromUsername string // grafanaPrometheusUsername is the username used to send data to Grafana Prometheus.
+ grafanaLokiUrl string // grafanaPostUrl is the URL used to send data to Grafana Loki.
+ grafanaLokiUsername string // grafanaLokiUsername is the username used to send data to Grafana Loki.
+ grafanaAccessToken string // grafanaAccessToken is the access token used to send data to Grafana.
+ newRelicLicenseKey string // newRelicKey is the key used to send data to New Relic.
+ newRelicMetricsUrl string // newRelicMetricsUrl is the URL used to send data to New Relic.
+ newRelicLogsUrl string // newRelicLogsUrl is the URL used to send logs to New Relic.
+ dataDogMetricsUrl string // dataDogMetricsUrl is the URL used to send data to DataDog.
+ dataDogLogsUrl string // dataDogLogsUrl is the URL used to send logs to DataDog.
+ dataDogAPIKey string // dataDogAPIKey is the API key used to send data to DataDog.
+)
+
+func normalizeString(s string) string {
+ // Remove backslashes
+ s = strings.ReplaceAll(s, `\`, "")
+
+ // Replace double quotes with single quotes
+ s = strings.ReplaceAll(s, `"`, "'")
+
+ // Normalize spacing around hyphens: no space before, one space after hyphens
+ s = strings.ReplaceAll(s, " - ", " -")
+ s = strings.ReplaceAll(s, " -", " -")
+ s = strings.ReplaceAll(s, "- ", " - ")
+
+ // Collapse multiple spaces into a single space
+ re := regexp.MustCompile(`\s+`)
+ s = re.ReplaceAllString(s, " ")
+
+ // Trim leading and trailing whitespace
+ s = strings.TrimSpace(s)
+
+ return s
+}
+
+func Init(cfg config.Configuration) error {
+ httpClient = &http.Client{Timeout: 5 * time.Second}
+ if cfg.ObservabilityPlatform.GrafanaCloud.LokiURL != "" {
+ grafanaPromUrl = cfg.ObservabilityPlatform.GrafanaCloud.PromURL
+ grafanaPromUsername = cfg.ObservabilityPlatform.GrafanaCloud.PromUsername
+ grafanaLokiUrl = cfg.ObservabilityPlatform.GrafanaCloud.LokiURL
+ grafanaLokiUsername = cfg.ObservabilityPlatform.GrafanaCloud.LokiUsername
+ grafanaAccessToken = cfg.ObservabilityPlatform.GrafanaCloud.AccessToken
+ } else if cfg.ObservabilityPlatform.NewRelic.Key != "" {
+ newRelicLicenseKey = cfg.ObservabilityPlatform.NewRelic.Key
+ newRelicMetricsUrl = cfg.ObservabilityPlatform.NewRelic.MetricsURL
+ newRelicLogsUrl = cfg.ObservabilityPlatform.NewRelic.LogsURL
+ } else if cfg.ObservabilityPlatform.DataDog.APIKey != "" {
+ dataDogMetricsUrl = cfg.ObservabilityPlatform.DataDog.MetricsURL
+ dataDogLogsUrl = cfg.ObservabilityPlatform.DataDog.LogsURL
+ dataDogAPIKey = cfg.ObservabilityPlatform.DataDog.APIKey
+ }
+ return nil
+}
+
+// SendToPlatform sends observability data to the appropriate platform.
+func SendToPlatform(data map[string]interface{}) {
+ if grafanaLokiUrl != "" {
+ if data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions" || data["endpoint"] == "cohere.generate" || data["endpoint"] == "cohere.chat" || data["endpoint"] == "cohere.summarize" || data["endpoint"] == "anthropic.completions" {
+ if data["finishReason"] == nil {
+ data["finishReason"] = "null"
+ }
+ metrics := []string{
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v completionTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["completionTokens"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v promptTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["promptTokens"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v totalTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["totalTokens"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["requestDuration"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["usageCost"]),
+ }
+ var metricsBody = []byte(strings.Join(metrics, "\n"))
+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)
+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")
+ }
+
+ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)
+
+ response_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"response\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), normalizeString(data["response"].(string))))
+ err = sendTelemetry(response_log, authHeader, grafanaLokiUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")
+ }
+
+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), normalizeString(data["prompt"].(string))))
+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")
+ }
+ } else if data["endpoint"] == "openai.embeddings" || data["endpoint"] == "cohere.embed" {
+ if data["endpoint"] == "openai.embeddings" {
+ metrics := []string{
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v promptTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["promptTokens"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v totalTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["totalTokens"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["requestDuration"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["usageCost"]),
+ }
+ var metricsBody = []byte(strings.Join(metrics, "\n"))
+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)
+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")
+ }
+
+ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)
+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["prompt"]))
+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")
+ }
+ } else {
+ metrics := []string{
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v promptTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["promptTokens"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["requestDuration"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["usageCost"]),
+ }
+ var metricsBody = []byte(strings.Join(metrics, "\n"))
+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)
+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")
+ }
+
+ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)
+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["prompt"]))
+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")
+ }
+ }
+ } else if data["endpoint"] == "openai.fine_tuning" {
+ metrics := []string{
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finetuneJobId=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finetuneJobId"], data["requestDuration"]),
+ }
+ var metricsBody = []byte(strings.Join(metrics, "\n"))
+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)
+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")
+ }
+ } else if data["endpoint"] == "openai.images.create" || data["endpoint"] == "openai.images.create.variations" {
+ metrics := []string{
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,imageSize=%v,imageQuality=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"], data["requestDuration"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,imageSize=%v,imageQuality=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"], data["usageCost"]),
+ }
+ var metricsBody = []byte(strings.Join(metrics, "\n"))
+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)
+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")
+ }
+
+ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)
+ if data["endpoint"] != "openai.images.create.variations" {
+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["revisedPrompt"]))
+ if data["model"] == "dall-e-2" {
+ prompt_log = []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["prompt"]))
+ }
+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")
+ }
+ }
+ image_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"image\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["image"]))
+ err = sendTelemetry(image_log, authHeader, grafanaLokiUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")
+ }
+ } else if data["endpoint"] == "openai.audio.speech.create" {
+ metrics := []string{
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,audioVoice=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"], data["requestDuration"]),
+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,audioVoice=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"], data["usageCost"]),
+ }
+ var metricsBody = []byte(strings.Join(metrics, "\n"))
+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)
+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")
+ }
+
+ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)
+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["prompt"]))
+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")
+ if err != nil {
+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")
+ }
+ }
+ } else if newRelicMetricsUrl != "" {
+ configureNewRelicData(data)
+ } else if dataDogMetricsUrl != "" {
+ configureDataDogData(data)
+ } else {
+ log.Info().Msg("No Observability Platform configured")
+ }
+}
+
+func sendTelemetry(telemetryData []byte, authHeader string, url string, requestType string) error {
+
+ req, err := http.NewRequest(requestType, url, bytes.NewBuffer(telemetryData))
+ if err != nil {
+ return fmt.Errorf("Error creating request")
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", authHeader)
+
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("Error sending request to %v", url)
+ } else if resp.StatusCode == 404 {
+ return fmt.Errorf("Provided URL %v is not valid", url)
+ } else if resp.StatusCode == 401 {
+ return fmt.Errorf("Provided credentials are not valid")
+ }
+
+ defer resp.Body.Close()
+
+ log.Info().Msgf("Successfully exported data to %v", url)
+ return nil
+}