Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
Add Ingester
patcher99d8072a82126e
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.ymlnew file mode 100644index 00000000..737811a0--- /dev/null+++ b/.github/workflows/docker-publish.yml@@ -0,0 +1,114 @@+name: Release++on:+ push:+ tags: [ '*.*.*' ]++env:+ REGISTRY: ghcr.io+ IMAGE_NAME: ${{ github.repository }}++permissions:+ contents: write++jobs:+ docker:+ runs-on: ubuntu-latest+ outputs:+ version: ${{ steps.set-version-output.outputs.version }}+ permissions:+ contents: write+ packages: write+ id-token: write++ steps:+ - name: Checkout repository+ uses: actions/checkout@v3++ - name: Install cosign+ uses: sigstore/cosign-installer@6e04d228eb30da1757ee4e1dd75a0ec73a653e06 #v3.1.1+ with:+ cosign-release: 'v2.1.1'++ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0+ with:+ platforms: linux/arm64/v8,linux/arm/v8++ - name: Code Format Check (gofmt)+ run: |+ if [ -n "$(gofmt -l .)" ]; then+ echo "Go code is not formatted:"+ gofmt -d .+ exit 1+ fi++ - name: Log into registry ${{ env.REGISTRY }}+ uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0+ with:+ registry: ${{ env.REGISTRY }}+ username: ${{ github.actor }}+ password: ${{ secrets.GITHUB_TOKEN }}++ - name: Extract Docker metadata+ id: meta+ uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0+ with:+ images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}++ - name: Build and push Docker image+ id: build-and-push+ uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0+ with:+ context: .+ push: ${{ github.event_name != 'pull_request' }}+ tags: ${{ steps.meta.outputs.tags }}+ labels: ${{ steps.meta.outputs.labels }}+ cache-from: type=gha+ cache-to: type=gha,mode=max++ - name: Sign the published Docker image+ env:+ TAGS: ${{ steps.meta.outputs.tags }}+ DIGEST: ${{ steps.build-and-push.outputs.digest }}+ run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}++ - name: Set version output+ id: set-version-output+ run: echo "::set-output name=version::${{ env.DOCKER_METADATA_OUTPUT_VERSION }}"++ binaries:+ name: Publish binaries+ needs: docker+ runs-on: ubuntu-latest+ strategy:+ matrix:+ platform:+ - {os: "linux", arch: "amd64"}+ - {os: "linux", arch: "arm64"}+ - {os: "linux", arch: "arm"}+ - {os: "linux", arch: "ppc64le"}+ - {os: "windows", arch: "amd64"}+ - {os: "windows", arch: "386"}+ - {os: "darwin", arch: "amd64"}+ - {os: "darwin", arch: "arm64"}++ steps:+ - uses: actions/checkout@v3++ - uses: actions/setup-go@v4+ with:+ go-version: '1.21.5'++ - name: Build package for ${{ matrix.platform.os }}-${{ matrix.platform.arch }}+ working-directory: ./src+ run: env GOOS=${{ matrix.platform.os }} GOARCH=${{ matrix.platform.arch }} go build -o ingester-${{ matrix.platform.os }}-${{ matrix.platform.arch }}++ - name: Upload binaries to release+ uses: svenstaro/upload-release-action@v2+ with:+ repo_token: ${{ secrets.GITHUB_TOKEN }}+ file: src/ingester-${{ matrix.platform.os }}-${{ matrix.platform.arch }}+ asset_name: ingester-${{ matrix.platform.os }}-${{ matrix.platform.arch }}+ tag: ${{ needs.docker.outputs.version }}+ overwrite: truediff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlnew file mode 100644index 00000000..ddea41be--- /dev/null+++ b/.github/workflows/tests.yml@@ -0,0 +1,37 @@+name: Tests++on:+ push:+ branches: [ "main" ]+ pull_request:+ branches: [ "main" ]++defaults:+ run:+ working-directory: src++jobs:++ build:+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v3++ - name: Set up Go+ uses: actions/setup-go@v4+ with:+ go-version: '1.21.1'++ - name: Build+ run: go build -v ./...++ - name: Code Format Check (gofmt)+ run: |+ if [ -n "$(gofmt -l .)" ]; then+ echo "Go code is not formatted:"+ gofmt -d .+ exit 1+ fi++ - name: Test+ run: go test -v ./...diff --git a/.gitignore b/.gitignoreindex b512c09d..610c8034 100644--- a/.gitignore+++ b/.gitignore@@ -1 +1,3 @@-node_modules\ No newline at end of file+node_modules+test.py+test.go\ No newline at end of filediff --git a/Dockerfile b/Dockerfilenew file mode 100644index 00000000..c7c2098b--- /dev/null+++ b/Dockerfile@@ -0,0 +1,17 @@+# Use the official Go image as the base image+FROM golang:1.21.5++# Set the working directory to /app+WORKDIR /app++COPY src/go.mod .+COPY src/go.sum .++# Download dependencies in a separate layer to leverage Docker cache+RUN go mod download++# Now copy the rest of your source code.+COPY src .++# Build the Go application and name the executable as "ingester"+RUN go build -o ingester\ No newline at end of filediff --git a/assets/example-config.yml b/assets/example-config.ymlnew file mode 100644index 00000000..c2d1ac8c--- /dev/null+++ b/assets/example-config.yml@@ -0,0 +1,41 @@+ingesterPort: 9044 # Port to run the Ingestion Server on, Example: 9044++# Comment out either 'localFile' or 'url' to use only one at a time.+pricingInfo:+ # localFile:+ # path: "/assets/pricing.json" # Path to local JSON file with LLM Pricing data+ url: "https://raw.githubusercontent.com/dokulabs/ingester/main/assets/pricing.json" # URL to download Pricing data file++# Configuration for the Doku Backend Database (TimescaleDB)+dbConfig:+ name: "DBNAME" # Name of the database, Example: "postgres"+ username: "USERNAME" # Username to connect to the database, Example: "admin"+ password: "PASSWORD" # Password to connect to the database, Example: "DOKU"+ host: "HOSTURL" # Hostname of the database, Example: "174.21.1.2"+ port: DBPORT # Port of the database, Example: 5432+ sslMode: SSLMODE # SSL Mode of the database, Example: "require"+ maxOpenConns: MAXOPENCONNS # Maximum number of open connections to the database, Example: 15+ maxIdleConns: MAXOPENCONNS # Maximum number of idle connections to the database, Example: 15+ dataTable: DATATABLE # Name of the table to store LLM Data, Example: "DOKU"+ apiKeyTable: APIKEYTABLE # Name of the table to store API Keys, Example: "APIKEYS"++# Configure Platform to export LLM Observability Data from Doku+# Only one platform can be enabled at a time, To enable a platform, set enabled to true and fill in the required fields for that platform.+observabilityPlatform:+ enabled: false # Enable or Disable the Observability Platform, Example: true+ # grafanaCloud:+ # promUrl: "influx-line-proxy-url" # URL to the Influx Line Proxy URL of the Grafana Cloud Prometheus Instance+ # promUsername: "prometheus-userid" # Prometheus User ID of the Grafana Cloud Prometheus Instance+ # lokiUrl: "loki-push-url" # URL to the Loki Push URL of the Grafana Cloud Loki Instance+ # lokiUsername: "loki-username" # Loki Username of the Grafana Cloud Loki Instance+ # accessToken: "grafana-cloud-access-token" # Access Token of the Grafana Cloud Instance++ # newRelic:+ # metricsUrl: "https://metric-api.newrelic.com/metric/v1" # URL to the New Relic Metric API+ # logsUrl: "https://log-api.newrelic.com/log/v1" # URL to the New Relic Log API+ # key: "newrelic-api-key" # Ingest API Key of the New Relic Account++ # datadog:+ # metricsUrl: "https://api.datadoghq.com/api/v1/series" # URL to the DataDog Metrics API+ # logsUrl: "https://http.logs.datadoghq.com/api/v2/logs" # URL to the DataDog Logs API+ # apiKey: "datadog-api-key" # API Key of the DataDog Account\ No newline at end of filediff --git a/assets/pricing.json b/assets/pricing.jsonnew file mode 100644index 00000000..a92d3830--- /dev/null+++ b/assets/pricing.json@@ -0,0 +1,145 @@+{+ "embeddings": {+ "text-embedding-ada-002": 0.0001,+ "ada": 0.0001,+ "text-ada-001": 0.0001,+ "embed-english-v3.0": 0.0001,+ "embed-multilingual-v3.0": 0.0001,+ "embed-english-light-v3.0": 0.0001,+ "embed-multilingual-light-v3.0": 0.0001,+ "embed-english-v2.0": 0.0001,+ "embed-english-light-v2.0": 0.0001,+ "embed-multilingual-v2.0": 0.0001+ },+ "images": {+ "dall-e-3": {+ "standard": {+ "1024x1024": 0.040,+ "1024x1792": 0.020,+ "1792x1024": 0.080+ },+ "hd": {+ "1024x1024": 0.080,+ "1024x1792": 0.120,+ "1792x1024": 0.120+ }+ },+ "dall-e-2": {+ "standard": {+ "1024x1024": 0.020,+ "512x512": 0.018,+ "256x256": 0.016+ }+ }+ },+ "audio": {+ "tts-1" : 0.015,+ "tts-1-hd": 0.030+ },+ "chat": {+ "ada": {+ "promptPrice": 0.0004,+ "completionPrice": 0.0004+ },+ "text-ada-001": {+ "promptPrice": 0.0004,+ "completionPrice": 0.0004+ },+ "babbage": {+ "promptPrice": 0.0005,+ "completionPrice": 0.0005+ },+ "babbage-002": {+ "promptPrice": 0.0004,+ "completionPrice": 0.0004+ },+ "text-babbage-001": {+ "promptPrice": 0.0005,+ "completionPrice": 0.0005+ },+ "curie": {+ "promptPrice": 0.0020,+ "completionPrice": 0.0020+ },+ "text-curie-001": {+ "promptPrice": 0.0020,+ "completionPrice": 0.0020+ },+ "davinci": {+ "promptPrice": 0.0200,+ "completionPrice": 0.0200+ },+ "davinci-002": {+ "promptPrice": 0.0020,+ "completionPrice": 0.0020+ },+ "text-davinci-001": {+ "promptPrice": 0.0200,+ "completionPrice": 0.0200+ },+ "text-davinci-002": {+ "promptPrice": 0.0200,+ "completionPrice": 0.0200+ },+ "text-davinci-003": {+ "promptPrice": 0.0200,+ "completionPrice": 0.0200+ },+ "gpt-3.5-turbo": {+ "promptPrice": 0.0010,+ "completionPrice": 0.0020+ },+ "gpt-3.5-turbo-instruct": {+ "promptPrice": 0.0015,+ "completionPrice": 0.0020+ },+ "gpt-4": {+ "promptPrice": 0.03,+ "completionPrice": 0.06+ },+ "gpt-4-32k": {+ "promptPrice": 0.06,+ "completionPrice": 0.12+ },+ "gpt-4-1106-preview": {+ "promptPrice": 0.01,+ "completionPrice": 0.03+ },+ "gpt-4-preview": {+ "promptPrice": 0.01,+ "completionPrice": 0.03+ },+ "gpt-4-1106-vision-preview": {+ "promptPrice": 0.01,+ "completionPrice": 0.03+ },+ "gpt-4-vision-preview": {+ "promptPrice": 0.01,+ "completionPrice": 0.03+ },+ "claude-instant-1": {+ "promptPrice": 0.00163,+ "completionPrice": 0.00551+ },+ "claude-2": {+ "promptPrice": 0.01102,+ "completionPrice": 0.03268+ },+ "command": {+ "promptPrice": 0.001,+ "completionPrice": 0.002+ },+ "command-nightly": {+ "promptPrice": 0.001,+ "completionPrice": 0.002+ },+ "command-light": {+ "promptPrice": 0.0003,+ "completionPrice": 0.0006+ },+ "command-light-nightly": {+ "promptPrice": 0.0003,+ "completionPrice": 0.0006+ }+ }+}\ No newline at end of filediff --git a/src/ingester/api/handlers.go b/src/ingester/api/handlers.gonew file mode 100644index 00000000..389ed828--- /dev/null+++ b/src/ingester/api/handlers.go@@ -0,0 +1,190 @@+package api++import (+ "encoding/json"+ "fmt"+ "net/http"+ "strings"++ "ingester/auth"+ "ingester/db"++ "github.com/rs/zerolog/log"+)++const (+ // Constants for error messages+ errMsgKeyExists = "An API Key with the name '%s' already exists"+ errMsgAuthFailed = "Unauthorized: Please check your API Key and try again"+ errMsgKeyNotFound = "Unable to find API Key with the given name %s"+ errMsgInvalidBody = "Invalid request body"+)++// APIKeyRequest represents the expected request structure for API Key related endpoints.+type APIKeyRequest struct {+ Name string `json:"name"`+}++// jsonResponse represents the expected response structure for all endpoints.+type jsonResponse struct {+ Status int `json:"status"`+ Message string `json:"message"`+ Data interface{} `json:"data,omitempty"`+}++// Normalize capitalizes names fields in the request body.+func (r *APIKeyRequest) Normalize() {+ r.Name = strings.ToLower(r.Name)+}++// decodeRequestBody decodes the JSON request body into the destination struct.+func decodeRequestBody(r *http.Request, dest interface{}) error {+ return json.NewDecoder(r.Body).Decode(dest)+}++// getAuthKey retrieves the API Key from the request header.+func getAuthKey(r *http.Request) string {+ return r.Header.Get("Authorization")+}++// sendJSONResponse constructs and sends a JSON response with appropriate headers.+func sendJSONResponse(w http.ResponseWriter, status int, message string) {+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(status)++ response := jsonResponse{+ Status: status,+ Message: message,+ }++ json.NewEncoder(w).Encode(response)+}++// handleAPIKeyErrors centralizes the error handling logic for API Key operations.+func handleAPIKeyErrors(w http.ResponseWriter, err error, name string) {+ if err.Error() == "KEYEXISTS" {+ sendJSONResponse(w, http.StatusConflict, fmt.Sprintf(errMsgKeyExists, name))+ return+ } else if err.Error() == "AUTHFAILED" {+ sendJSONResponse(w, http.StatusUnauthorized, errMsgAuthFailed)+ return+ } else if err.Error() == "NOTFOUND" {+ sendJSONResponse(w, http.StatusNotFound, fmt.Sprintf(errMsgKeyNotFound, name))+ return+ } else {+ sendJSONResponse(w, http.StatusInternalServerError, err.Error())+ return+ }+}++// generateAPIKeyHandler handles the creation of a new API Key+func generateAPIKeyHandler(w http.ResponseWriter, r *http.Request) {+ var request APIKeyRequest++ if err := decodeRequestBody(r, &request); err != nil {+ sendJSONResponse(w, http.StatusBadRequest, errMsgInvalidBody)+ return+ }+ request.Normalize()++ newAPIKey, err := db.GenerateAPIKey(getAuthKey(r), request.Name)+ if err != nil {+ handleAPIKeyErrors(w, err, request.Name)+ return+ }++ sendJSONResponse(w, http.StatusOK, newAPIKey)+}++// getAPIKeyHandler handles retrieving an existing API key.+func getAPIKeyHandler(w http.ResponseWriter, r *http.Request) {+ var request APIKeyRequest++ if err := decodeRequestBody(r, &request); err != nil {+ sendJSONResponse(w, http.StatusBadRequest, errMsgInvalidBody)+ return+ }+ request.Normalize()++ apiKey, err := db.GetAPIKeyForName(getAuthKey(r), request.Name)+ if err != nil {+ handleAPIKeyErrors(w, err, request.Name)+ return+ }++ sendJSONResponse(w, http.StatusOK, apiKey)+}++// deleteAPIKeyHandler handles deleting an existing API key.+func deleteAPIKeyHandler(w http.ResponseWriter, r *http.Request) {+ var request APIKeyRequest++ if err := decodeRequestBody(r, &request); err != nil {+ sendJSONResponse(w, http.StatusBadRequest, errMsgInvalidBody)+ return+ }++ request.Normalize()+ err := db.DeleteAPIKey(getAuthKey(r), request.Name)+ if err != nil {+ handleAPIKeyErrors(w, err, request.Name)+ return+ }++ sendJSONResponse(w, http.StatusOK, "API key deleted successfully")+}++// DataHandler handles data related operations recieved on `/api/push` endpoint.+func DataHandler(w http.ResponseWriter, r *http.Request) {+ var data map[string]interface{}++ if err := decodeRequestBody(r, &data); err != nil {+ sendJSONResponse(w, http.StatusBadRequest, errMsgInvalidBody)+ return+ }++ var err error+ data["name"], err = auth.AuthenticateRequest(getAuthKey(r))+ if err != nil {+ handleAPIKeyErrors(w, err, "")+ return+ }++ // Check if skipResp is true+ skipResp, ok := data["skipResp"].(bool)+ if ok && skipResp == true {+ sendJSONResponse(w, http.StatusAccepted, "Insertion started in background")+ go db.PerformDatabaseInsertion(data) // Running as a goroutine for async processing+ return+ }++ responseMessage, statusCode := db.PerformDatabaseInsertion(data)++ // Respond to the user with the insertion status and any potential errors+ sendJSONResponse(w, statusCode, responseMessage)+}++// APIKeyHandler handles all API Key tasks recieved on `/api/keys` endpoint.+func APIKeyHandler(w http.ResponseWriter, r *http.Request) {+ switch r.Method {+ case "GET":+ getAPIKeyHandler(w, r)+ case "POST":+ generateAPIKeyHandler(w, r)+ case "DELETE":+ deleteAPIKeyHandler(w, r)+ default:+ w.WriteHeader(http.StatusMethodNotAllowed)+ }+}++// BaseEndpoint serves as a health check and entry point for the service.+func BaseEndpoint(w http.ResponseWriter, r *http.Request) {+ if err := db.PingDB(); err != nil {+ log.Error().Err(err).Msgf("Health check failed") // Log the error+ sendJSONResponse(w, http.StatusServiceUnavailable, "Database is currently not reachable from the server")+ return+ }+ // The database is up and reachable.+ sendJSONResponse(w, http.StatusOK, "Welcome to Doku Ingester - Service operational")+}diff --git a/src/ingester/auth/auth.go b/src/ingester/auth/auth.gonew file mode 100644index 00000000..102da29a--- /dev/null+++ b/src/ingester/auth/auth.go@@ -0,0 +1,78 @@+package auth++import (+ "database/sql"+ "fmt"+ "sync"+ "time"++ "ingester/db"++ "github.com/rs/zerolog/log"+)++var (+ // apiKeyCache stores the lookup of API keys and organization IDs.+ apiKeyCache = sync.Map{}+ // CacheEntryDuration defines how long an item should stay in the cache before being re-validated.+ CacheEntryDuration = time.Minute * 10+)++// cacheEntry represents an entry in the API key cache.+type cacheEntry struct {+ Name string+ Timestamp time.Time+}++// InitializeCacheEviction sets up periodic eviction of expired cache entries.+func InitializeCacheEviction() {+ go func() {+ ticker := time.NewTicker(CacheEntryDuration)+ defer ticker.Stop()++ for {+ <-ticker.C // This blocks until the ticker sends a value+ log.Info().Msg("Evicting Expired API Key Cache Entries")+ evictExpiredEntries()+ }+ }()+}++// EvictExpiredEntries goes through the cache and evicts expired entries.+func evictExpiredEntries() {+ now := time.Now()+ apiKeyCache.Range(func(key, value interface{}) bool {+ if entry, ok := value.(cacheEntry); ok {+ if now.Sub(entry.Timestamp) >= CacheEntryDuration {+ apiKeyCache.Delete(key)+ }+ }+ return true+ })+}++// AuthenticateRequest checks the provided API key against the known keys.+func AuthenticateRequest(apiKey string) (string, error) {+ /// Attempt to retrieve API Key from the cache.+ if val, ok := apiKeyCache.Load(apiKey); ok {+ entry := val.(cacheEntry)+ if time.Since(entry.Timestamp) < CacheEntryDuration {+ return entry.Name, nil+ }+ }++ // If the key is not in the cache or the cache has expired, call the db to check the API key.+ name, err := db.CheckAPIKey(apiKey)+ if err != nil {+ if err == sql.ErrNoRows {+ log.Warn().Msg("Authorization Failed for an API Key")+ return "", fmt.Errorf("AUTHFAILED")+ }+ return "", err+ }++ // The API key has been successfully authenticated, so cache it.+ apiKeyCache.Store(apiKey, cacheEntry{Name: name, Timestamp: time.Now()})++ return name, nil+}diff --git a/src/ingester/config/config.go b/src/ingester/config/config.gonew file mode 100644index 00000000..07a7cb9a--- /dev/null+++ b/src/ingester/config/config.go@@ -0,0 +1,130 @@+package config++import (+ "fmt"+ "os"+ "strconv"++ "github.com/rs/zerolog/log"+ "gopkg.in/yaml.v2"+)++type Configuration struct {+ IngesterPort string `yaml:"ingesterPort"`+ PricingInfo struct {+ LocalFile struct {+ Path string `yaml:"path"`+ } `yaml:"localFile"`+ URL string `yaml:"url"`+ } `yaml:"pricingInfo"`+ DBConfig struct {+ DBName string `yaml:"name"`+ DBUser string `yaml:"username"`+ DBPassword string `yaml:"password"`+ DBHost string `yaml:"host"`+ DBPort string `yaml:"port"`+ DBSSLMode string `yaml:"sslMode"`+ MaxOpenConns int `yaml:"maxOpenConns"`+ MaxIdleConns int `yaml:"maxIdleConns"`+ DataTableName string `yaml:"dataTable"`+ APIKeyTableName string `yaml:"apiKeyTable"`+ } `yaml:"dbConfig"`+ ObservabilityPlatform struct {+ Enabled bool `yaml:"enabled"`+ GrafanaCloud struct {+ PromURL string `yaml:"promUrl"`+ PromUsername string `yaml:"promUsername"`+ LokiURL string `yaml:"lokiUrl"`+ LokiUsername string `yaml:"lokiUsername"`+ AccessToken string `yaml:"accessToken"`+ } `yaml:"grafanaCloud"`+ NewRelic struct {+ Key string `yaml:"key"`+ MetricsURL string `yaml:"metricsUrl"`+ LogsURL string `yaml:"logsUrl"`+ } `yaml:"newRelic"`+ DataDog struct {+ MetricsURL string `yaml:"metricsUrl"`+ LogsURL string `yaml:"logsUrl"`+ APIKey string `yaml:"apiKey"`+ } `yaml:"datadog"`+ } `yaml:"observabilityPlatform"`+}++func validateConfig(cfg *Configuration) error {+ if _, err := strconv.Atoi(cfg.IngesterPort); err != nil {+ return fmt.Errorf("Ingester Port is not defined")+ }++ // Check if at least one PricingInfo configuration is set.+ if cfg.PricingInfo.LocalFile.Path == "" && cfg.PricingInfo.URL == "" {+ return fmt.Errorf("PricingInfo configuration is not defined")+ }++ // Check if both PricingInfo configurations are set.+ if cfg.PricingInfo.LocalFile.Path != "" && cfg.PricingInfo.URL != "" {+ return fmt.Errorf("Both LocalFile and URL configurations are defined in PricingInfo; only one is allowed")+ }++ if cfg.DBConfig.DBPassword == "" {+ log.Info().Msg("'dbConfig.password' is not defined, trying to read from environment variable 'DB_PASSWORD'")+ cfg.DBConfig.DBPassword = os.Getenv("DB_PASSWORD")+ if cfg.DBConfig.DBPassword == "" {+ return fmt.Errorf("'DB_PASSWORD' environment variable is not set and 'dbConfig.password' is not defined in configuration")+ }+ log.Info().Msg("dbConfig.password is now set")+ }++ if cfg.DBConfig.DBUser == "" {+ log.Info().Msg("'dbConfig.username' is not defined, trying to read from environment variable 'DB_USERNAME")+ cfg.DBConfig.DBUser = os.Getenv("DB_USERNAME")+ if cfg.DBConfig.DBUser == "" {+ return fmt.Errorf("'DB_USERNAME' environment variable is not set and 'dbConfig.username' is not defined in configuration")+ }+ log.Info().Msg("dbConfig.username is now set")+ }++ if cfg.ObservabilityPlatform.Enabled {+ definedConfigs := 0++ if cfg.ObservabilityPlatform.GrafanaCloud.PromURL != "" {+ definedConfigs+++ }+ if cfg.ObservabilityPlatform.NewRelic.Key != "" {+ definedConfigs+++ }+ if cfg.ObservabilityPlatform.DataDog.APIKey != "" {+ definedConfigs+++ }++ if definedConfigs > 1 {+ return fmt.Errorf("Only one observability platform configuration (GrafanaCloud, NewRelic, or DataDog) can be enabled at a time")+ }++ if definedConfigs == 0 {+ return fmt.Errorf("Observability platform is enabled, but no observability configurations are defined")+ }+ }++ return nil+}++func LoadConfiguration(configPath string) (*Configuration, error) {+ data, err := os.ReadFile(configPath)+ if err != nil {+ return nil, err+ }++ var cfg Configuration+ err = yaml.Unmarshal(data, &cfg)+ if err != nil {+ return nil, err+ }++ // Validate the loaded configuration+ if err := validateConfig(&cfg); err != nil {+ return nil, err+ }++ return &cfg, nil+}diff --git a/src/ingester/cost/cost.go b/src/ingester/cost/cost.gonew file mode 100644index 00000000..4539850e--- /dev/null+++ b/src/ingester/cost/cost.go@@ -0,0 +1,163 @@+package cost++import (+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "os"+)++var Pricing PricingModel++// PricingModel is the pricing information for the different models and features.+type PricingModel struct {+ Embeddings map[string]float64 `json:"embeddings"`+ Images map[string]map[string]map[string]float64 `json:"images"`+ Audio map[string]float64 `json:"audio"`+ Chat map[string]struct {+ PromptPrice float64 `json:"promptPrice"`+ CompletionPrice float64 `json:"completionPrice"`+ } `json:"chat"`+}++// validatePricingData validates the pricing data for the different models and features.+func validatePricingData(pricingModel PricingModel) error {+ // Example validation for Embeddings pricing+ if len(pricingModel.Embeddings) == 0 {+ return fmt.Errorf("Embeddings pricing data is not defined")+ }++ // Example validation for Audio pricing+ if len(pricingModel.Audio) == 0 {+ return fmt.Errorf("Audio pricing data is not defined")+ }++ // Validate the Images pricing, which has nested maps+ for model, qualityMap := range pricingModel.Images {+ if len(qualityMap) == 0 {+ return fmt.Errorf("image pricing data for model '%s' is not defined in the JSON File", model)+ }+ for quality, sizeMap := range qualityMap {+ if len(sizeMap) == 0 {+ return fmt.Errorf("image pricing data for model '%s', quality '%s' is not defined in the JSON File", model, quality)+ }+ }+ }++ // Validate the Chat pricing+ for model, chatPricing := range pricingModel.Chat {+ if chatPricing.PromptPrice == 0 {+ return fmt.Errorf("Prompt Tokens pricing data for model '%s' is not defined in the JSON File", model)+ } else if chatPricing.CompletionPrice == 0 {+ return fmt.Errorf("Completion Tokens pricing data for model '%s' is not defined in the JSON File", model)+ }+ }++ return nil+}++// fetchJSONFromURL is a new function for fetching JSON content from a URL.+func fetchJSONFromURL(url string) ([]byte, error) {+ maxRetries := 5+ var resp *http.Response+ var err error+ for i := 0; i < maxRetries; i++ {+ resp, err = http.Get(url)+ if err == nil {+ break+ }+ }++ if err != nil {+ return nil, fmt.Errorf("Failed to make request to URL %s", url)+ }+ defer resp.Body.Close()++ if resp.StatusCode != http.StatusOK {+ return nil, fmt.Errorf("Failed to fetch JSON content from URL %s", url)+ }++ return io.ReadAll(resp.Body)+}++// fetchJSONFromFile is a new function for fetching JSON content from a URL.+func fetchJSONFromFile(path string) ([]byte, error) {+ content, err := os.ReadFile(path)+ if err != nil {+ return nil, fmt.Errorf("Failed to fetch JSON content from file '%s'", path)+ }++ return content, nil+}++// LoadPricing loads the pricing information from the given file.+func LoadPricing(path, url string) error {+ var content []byte+ var err error++ switch {+ case path != "":+ content, err = fetchJSONFromFile(path)+ case url != "":+ content, err = fetchJSONFromURL(url)+ }+ if err != nil {+ return err+ }++ if err = json.Unmarshal(content, &Pricing); err != nil {+ return fmt.Errorf("Failed to unmarshal costing JSON: %w", err)+ }++ if err = validatePricingData(Pricing); err != nil {+ return err+ }++ return nil+}++// calculateEmbeddingsCost calculates the cost for embeddings based on the model and prompt tokens.+func CalculateEmbeddingsCost(promptTokens float64, model string) (float64, error) {+ price, ok := Pricing.Embeddings[model]+ if !ok {+ return 0, nil+ }+ return (promptTokens / 1000) * price, nil+}++// calculateImageCost calculates the cost for images based on the model, image size, and quality.+func CalculateImageCost(model, imageSize, quality string) (float64, error) {+ models, ok := Pricing.Images[model]+ if !ok {+ return 0, nil+ }+ qualities, ok := models[quality]+ if !ok {+ return 0, nil+ }+ price, ok := qualities[imageSize]+ if !ok {+ return 0, nil+ }++ return price, nil+}++// calculateChatCost calculates the cost for chat based on the model, prompt tokens, and completion tokens.+func CalculateChatCost(promptTokens, completionTokens float64, model string) (float64, error) {+ chatModel, ok := Pricing.Chat[model]+ if !ok {+ return 0, nil+ }+ return ((promptTokens / 1000) * chatModel.PromptPrice) + ((completionTokens / 1000) * chatModel.CompletionPrice), nil+}++// CalculateAudioCost calculates the cost for Audio based on the model, and prompt.+func CalculateAudioCost(prompt string, model string) (float64, error) {+ price, ok := Pricing.Audio[model]+ if !ok {+ return 0, nil+ }+ return ((float64(len(prompt)) / 1000) * price), nil+}diff --git a/src/ingester/db/db.go b/src/ingester/db/db.gonew file mode 100644index 00000000..13a8f9d2--- /dev/null+++ b/src/ingester/db/db.go@@ -0,0 +1,437 @@+package db++import (+ "crypto/rand"+ "database/sql"+ "encoding/hex"+ "fmt"+ "ingester/config"+ "ingester/cost"+ "ingester/obsPlatform"+ "net/http"+ "sync"++ _ "github.com/lib/pq"+ "github.com/pkoukk/tiktoken-go"+ "github.com/rs/zerolog/log"+)++var (+ once sync.Once // once is used to ensure that the database is initialized only once+ db *sql.DB // db holds the database connection+ dbConfig DatabaseConfig // dbConfig holds the database configuration++ // validFields represent the fields that are expected in the incoming data.+ validFields = []string{+ "name",+ "environment",+ "endpoint",+ "sourceLanguage",+ "applicationName",+ "completionTokens",+ "promptTokens",+ "totalTokens",+ "finishReason",+ "requestDuration",+ "usageCost",+ "model",+ "prompt",+ "response",+ "imageSize",+ "revisedPrompt",+ "image",+ "audioVoice",+ "finetuneJobId",+ "finetuneJobStatus",+ }+)++// DBConfig holds the database configuration+type DatabaseConfig struct {+ DBName string+ User string+ Password string+ Host string+ Port string+ SSLMode string+ MaxIdleConns int+ MaxOpenConns int+ DataTableName string+ ApiKeyTableName string+}++// PingDB attempts to ping the database to check if it's alive.+func PingDB() error {+ return db.Ping()+}++// GenerateSecureRandomKey should generate a secure random string to be used as an API key.+func generateSecureRandomKey() (string, error) {+ randomPartLength := 40 / 2 // Each byte becomes two hex characters, so we need half as many bytes.++ randomBytes := make([]byte, randomPartLength)+ _, err := rand.Read(randomBytes)+ if err != nil {+ log.Error().Err(err).Msg("Error generating random bytes")+ // In the case of an error, return it as we cannot generate a key.+ return "", err+ }++ // Encode the random bytes as a hex string and prefix with 'dk'.+ randomHexString := hex.EncodeToString(randomBytes)+ apiKey := "dk" + randomHexString++ return apiKey, nil+}++// getCreateAPIKeysTableSQL returns the SQL query to create the API keys table.+func getCreateAPIKeysTableSQL(tableName string) string {+ return fmt.Sprintf(`+ CREATE TABLE IF NOT EXISTS %s (+ id SERIAL PRIMARY KEY,+ api_key VARCHAR(255) NOT NULL UNIQUE,+ name VARCHAR(50) NOT NULL+ );`, tableName)+}++// getCreateDataTableSQL returns the SQL query to create the data table.+func getCreateDataTableSQL(tableName string) string {+ return fmt.Sprintf(`+ CREATE TABLE IF NOT EXISTS %s (+ time TIMESTAMPTZ NOT NULL,+ name VARCHAR(10) NOT NULL,+ environment VARCHAR(50) NOT NULL,+ endpoint VARCHAR(50) NOT NULL,+ sourceLanguage VARCHAR(50) NOT NULL,+ applicationName VARCHAR(50) NOT NULL,+ completionTokens INTEGER,+ promptTokens INTEGER,+ totalTokens INTEGER,+ finishReason VARCHAR(50),+ requestDuration DOUBLE PRECISION,+ usageCost DOUBLE PRECISION,+ model VARCHAR(50),+ prompt TEXT,+ response TEXT,+ imageSize TEXT,+ revisedPrompt TEXT,+ image TEXT,+ audioVoice TEXT,+ finetuneJobId TEXT,+ finetuneJobStatus TEXT+ );`, tableName)+}++// tableExists checks if a table exists in the database.+func tableExists(db *sql.DB, tableName string) (bool, error) {+ query := `+ SELECT EXISTS (+ SELECT 1+ FROM information_schema.tables+ WHERE table_schema = 'public'+ AND lower(table_name) = lower($1)+ )+ `++ var exists bool+ err := db.QueryRow(query, tableName).Scan(&exists)+ if err != nil {+ return false, err+ }++ return exists, nil+}++// createTable creates a table in the database if it doesn't exist.+func createTable(db *sql.DB, tableName string) error {+ var createTableSQL string+ if tableName == dbConfig.ApiKeyTableName {+ createTableSQL = getCreateAPIKeysTableSQL(tableName)+ } else if tableName == dbConfig.DataTableName {+ createTableSQL = getCreateDataTableSQL(tableName)+ }++ exists, err := tableExists(db, tableName)+ if err != nil {+ return fmt.Errorf("Error checking table '%s' existence: %w", tableName, err)+ }+ if exists == false {+ _, err := db.Exec(createTableSQL)+ if err != nil {+ return fmt.Errorf("Error creating table %s: %w", tableName, err)+ }+ log.Info().Msgf("Table '%s' created in the database", tableName)++ if tableName == dbConfig.ApiKeyTableName {+ createIndexSQL := fmt.Sprintf("CREATE INDEX IF NOT EXISTS idx_api_key ON %s (api_key);", tableName)+ _, err = db.Exec(createIndexSQL)+ if err != nil {+ return fmt.Errorf("Error creating index on 'api_key' column: %w", err)+ }+ log.Info().Msgf("Index on 'api_key' column checked/created in table '%s'", tableName)+ }++ // If the table to create is the data table, convert it into a hypertable+ if tableName == dbConfig.DataTableName {+ _, err := db.Exec("SELECT create_hypertable($1, 'time')", tableName)+ if err != nil {+ return fmt.Errorf("Error creating hypertable: %w", err)+ }+ log.Info().Msgf("Table '%s' converted to a Hypertable", tableName)+ }+ } else {+ log.Info().Msgf("Table '%s' already exists in the database", tableName)+ }+ return nil+}++// initializeDB initializes connection to the database.+func initializeDB() error {+ var dbErr error+ once.Do(func() {+ connStr := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",+ dbConfig.Host, dbConfig.Port, dbConfig.User, dbConfig.Password, dbConfig.DBName, dbConfig.SSLMode)++ if db, dbErr = sql.Open("postgres", connStr); dbErr != nil {+ log.Error().Err(dbErr).Msg("Error connecting to the database")+ return+ }++ dbErr = db.Ping()+ if dbErr != nil {+ return+ }++ db.SetMaxOpenConns(dbConfig.MaxOpenConns)+ db.SetMaxIdleConns(dbConfig.MaxIdleConns)+ })+ return dbErr+}++func getTokens(text, model string) int {+ tkm, err := tiktoken.EncodingForModel(model)+ if err != nil {+ tkm, _ = tiktoken.GetEncoding("cl100k_base")+ }+ token := tkm.Encode(text, nil, nil)+ return len(token)+}++// insertDataToDB inserts data into the database.+func insertDataToDB(data map[string]interface{}) (string, int) {+ // Calculate usage cost based on the endpoint type+ if data["endpoint"] == "openai.embeddings" || data["endpoint"] == "cohere.embed" {+ data["usageCost"], _ = cost.CalculateEmbeddingsCost(data["promptTokens"].(float64), data["model"].(string))+ } else if data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions" || data["endpoint"] == "cohere.chat" || data["endpoint"] == "cohere.summarize" || data["endpoint"] == "cohere.generate" {+ if data["completionTokens"] != nil && data["promptTokens"] != nil {+ data["usageCost"], _ = cost.CalculateChatCost(data["promptTokens"].(float64), data["completionTokens"].(float64), data["model"].(string))+ } else if (data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions") && data["prompt"] != nil && data["response"] != nil {+ data["promptTokens"] = getTokens(data["prompt"].(string), data["model"].(string))+ data["completionTokens"] = getTokens(data["response"].(string), data["model"].(string))+ data["totalTokens"] = data["promptTokens"].(int) + data["completionTokens"].(int)+ data["usageCost"], _ = cost.CalculateChatCost(float64(data["promptTokens"].(int)), float64(data["completionTokens"].(int)), data["model"].(string))+ }+ } else if data["endpoint"] == "openai.images.create" || data["endpoint"] == "openai.images.create.variations" {+ data["usageCost"], _ = cost.CalculateImageCost(data["model"].(string), data["imageSize"].(string), data["imageQuality"].(string))+ } else if data["endpoint"] == "openai.audio.speech.create" {+ data["usageCost"], _ = cost.CalculateAudioCost(data["prompt"].(string), data["model"].(string))+ }++ // Fill missing fields with nil+ for _, field := range validFields {+ if _, exists := data[field]; !exists {+ data[field] = nil+ }+ }++ go obsPlatform.SendToPlatform(data)++ // Define the SQL query for data insertion+ query := fmt.Sprintf("INSERT INTO %s (time, name, environment, endpoint, sourceLanguage, applicationName, completionTokens, promptTokens, totalTokens, finishReason, requestDuration, usageCost, model, prompt, response, imageSize, revisedPrompt, image, audioVoice, finetuneJobId, finetuneJobStatus) VALUES (NOW(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)", dbConfig.DataTableName)++ // Execute the SQL query+ _, err := db.Exec(query,+ data["name"],+ data["environment"],+ data["endpoint"],+ data["sourceLanguage"],+ data["applicationName"],+ data["completionTokens"],+ data["promptTokens"],+ data["totalTokens"],+ data["finishReason"],+ data["requestDuration"],+ data["usageCost"],+ data["model"],+ data["prompt"],+ data["response"],+ data["imageSize"],+ data["revisedPrompt"],+ data["image"],+ data["audioVoice"],+ data["finetuneJobId"],+ data["finetuneJobStatus"],+ )+ if err != nil {+ log.Error().Err(err).Msg("Error Inserting data into the database")+ // Update the response message and status code for error+ return "Internal Server Error", http.StatusInternalServerError+ }++ return "Data insertion completed", http.StatusCreated+}++// Init initializes the database connection and creates the required tables.+func Init(cfg config.Configuration) error {++ // Initialize the database configuration+ dbConfig = DatabaseConfig{+ DBName: cfg.DBConfig.DBName,+ User: cfg.DBConfig.DBUser,+ Password: cfg.DBConfig.DBPassword,+ Host: cfg.DBConfig.DBHost,+ Port: cfg.DBConfig.DBPort,+ SSLMode: cfg.DBConfig.DBSSLMode,+ MaxIdleConns: cfg.DBConfig.MaxIdleConns,+ MaxOpenConns: cfg.DBConfig.MaxOpenConns,+ DataTableName: cfg.DBConfig.DataTableName,+ ApiKeyTableName: cfg.DBConfig.APIKeyTableName,+ }++ err := initializeDB()+ if err != nil {+ log.Error().Err(err).Msg("Error initializing database")+ return fmt.Errorf("Could not initialize connection to the database: %w", err)+ }++ // Create the DATA and API keys table if it doesn't exist.+ log.Info().Msgf("Creating '%s' and '%s' tables in the database if they don't exist", dbConfig.ApiKeyTableName, dbConfig.DataTableName)++ err = createTable(db, dbConfig.ApiKeyTableName)+ if err != nil {+ log.Error().Err(err).Msgf("Error creating table %s", dbConfig.ApiKeyTableName)+ return err+ }++ err = createTable(db, dbConfig.DataTableName)+ if err != nil {+ log.Error().Err(err).Msgf("Error creating table %s", dbConfig.ApiKeyTableName)+ return err+ }+ return nil+}++// PerformDatabaseInsertion performs the database insertion synchronously.+func PerformDatabaseInsertion(data map[string]interface{}) (string, int) {+ // Call insertDataToDB directly instead of starting a new goroutine.+ responseMessage, statusCode := insertDataToDB(data)++ // The operation is now synchronous. Once insertDataToDB returns, the result is ready to use.+ return responseMessage, statusCode+}++// CheckAPIKey retrieves the name associated with the given API key from the database.+func CheckAPIKey(apiKey string) (string, error) {+ var name string++ query := fmt.Sprintf("SELECT name FROM %s WHERE api_key = $1", dbConfig.ApiKeyTableName)+ err := db.QueryRow(query, apiKey).Scan(&name)+ if err != nil {+ if err == sql.ErrNoRows {+ return "", err+ }+ return "", err+ }+ return name, nil+}++// GenerateAPIKey generates a new API key for a given name and stores it in the database.+func GenerateAPIKey(existingAPIKey, name string) (string, error) {+ // If there are any existing API keys, authenticate the provided API key before proceeding+ var count int+ countQuery := fmt.Sprintf("SELECT COUNT(*) FROM %s", dbConfig.ApiKeyTableName)+ err := db.QueryRow(countQuery).Scan(&count)+ if err != nil {+ log.Error().Err(err).Msg("Error checking API key table")+ return "", fmt.Errorf("Failed to check API key table: %v", err)+ }++ // Only perform the check if the count is greater than zero+ if count > 0 {+ // Attempt to retrieve any existing key for the given name.+ _, err = GetAPIKeyForName(existingAPIKey, name)+ if err == nil {+ log.Warn().Msgf("Error creating new API Key as a key with the name '%s' already exists", name)+ return "", fmt.Errorf("KEYEXISTS")+ } else if err.Error() == "AUTHFAILED" {+ return "", err+ }+ }++ // No existing key found, proceed to generate a new API key+ log.Info().Msgf("Creating a new API Key with the name: %s", name)+ newAPIKey, _ := generateSecureRandomKey()++ // Insert the new API key into the database+ insertQuery := fmt.Sprintf("INSERT INTO %s (api_key, name) VALUES ($1, $2)", dbConfig.ApiKeyTableName)+ _, err = db.Exec(insertQuery, newAPIKey, name)+ if err != nil {+ log.Error().Err(err).Msg("Error inserting the new API key in the database")+ return "", err+ }+ log.Info().Msgf("API Key with the name '%s' created successfully", name)+ return newAPIKey, nil+}++// GetAPIKeyForName retrieves an API key for a given name from the database.+func GetAPIKeyForName(existingAPIKey, name string) (string, error) {++ // Autheticate the provided API key before proceeding+ _, err := CheckAPIKey(existingAPIKey)+ if err != nil {+ log.Warn().Msg("Authorization Failed for an API Key")+ return "", fmt.Errorf("AUTHFAILED")+ }++ // Retrieve the API key for the given name+ var apiKey string+ query := fmt.Sprintf("SELECT api_key FROM %s WHERE name = $1", dbConfig.ApiKeyTableName)+ err = db.QueryRow(query, name).Scan(&apiKey)+ if err != nil {+ if err == sql.ErrNoRows {+ log.Warn().Msgf("API Key with the name '%s' currently not found in the database", name)+ return "", fmt.Errorf("NOTFOUND")+ }+ log.Warn().Err(err).Msgf("Error retrieving API key for the name '%s'", name)+ return "", err+ }++ return apiKey, nil+}++// DeleteAPIKey deletes an API key for a given name from the database.+func DeleteAPIKey(existingAPIKey, name string) error {++ // Autheticate the provided API key before proceeding and check if the API key exists+ apiKey, err := GetAPIKeyForName(existingAPIKey, name)+ if err != nil {+ if err.Error() == "AUTHFAILED" {+ return err+ }+ if err.Error() == "NOTFOUND" {+ return err+ }+ return err+ }++ // Delete the API key from the database+ log.Info().Msgf("Deleting API Key with the name '%s' from the database", name)+ query := fmt.Sprintf("DELETE FROM %s WHERE api_key = $1", dbConfig.ApiKeyTableName)+ _, err = db.Exec(query, apiKey)+ if err != nil {+ log.Error().Err(err).Msg("Error deleting API key")+ return err+ }+ log.Info().Msgf("API Key with the name '%s' deleted successfully", name)+ return nil+}diff --git a/src/ingester/go.mod b/src/ingester/go.modnew file mode 100644index 00000000..16b21074--- /dev/null+++ b/src/ingester/go.mod@@ -0,0 +1,20 @@+module ingester++go 1.21.5++require (+ github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be+ github.com/gorilla/mux v1.8.1+ github.com/lib/pq v1.10.9+ github.com/pkoukk/tiktoken-go v0.1.6+ github.com/rs/zerolog v1.31.0+ gopkg.in/yaml.v2 v2.4.0+)++require (+ github.com/dlclark/regexp2 v1.10.0 // indirect+ github.com/google/uuid v1.3.0 // indirect+ github.com/mattn/go-colorable v0.1.13 // indirect+ github.com/mattn/go-isatty v0.0.19 // indirect+ golang.org/x/sys v0.12.0 // indirect+)diff --git a/src/ingester/go.sum b/src/ingester/go.sumnew file mode 100644index 00000000..37120922--- /dev/null+++ b/src/ingester/go.sum@@ -0,0 +1,39 @@+github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be h1:J5BL2kskAlV9ckgEsNQXscjIaLiOYiZ75d4e94E6dcQ=+github.com/common-nighthawk/go-figure v0.0.0-20210622060536-734e95fb86be/go.mod h1:mk5IQ+Y0ZeO87b858TlA645sVcEcbiX6YqP98kt+7+w=+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=+github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=+github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=+github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A=+github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=diff --git a/src/ingester/main.go b/src/ingester/main.gonew file mode 100644index 00000000..fb7acbde--- /dev/null+++ b/src/ingester/main.go@@ -0,0 +1,123 @@+package main++import (+ "context"+ "flag"+ "net/http"+ "os"+ "os/signal"+ "syscall"+ "time"++ "ingester/api"+ "ingester/auth"+ "ingester/config"+ "ingester/cost"+ "ingester/db"+ "ingester/obsPlatform"++ "github.com/common-nighthawk/go-figure"+ "github.com/gorilla/mux"+ "github.com/rs/zerolog"+ "github.com/rs/zerolog/log"+)++func waitForShutdown(server *http.Server) {+ quit := make(chan os.Signal, 1)+ signal.Notify(quit, os.Interrupt, syscall.SIGTERM)++ // Wait for an interrupt+ sig := <-quit+ log.Info().Msgf("Caught sig: %+v", sig)+ log.Info().Msg("Starting to shutdown server")++ // Initialize the context with a timeout to ensure the app can make a graceful exit+ // or abort if it takes too long+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()++ if err := server.Shutdown(ctx); err != nil {+ log.Error().Err(err).Msg("Server shutdown failed")+ } else {+ log.Info().Msg("Server gracefully shutdown")+ }+}++// main is the entrypoint for the Doku Ingester service. It sets up logging,+// initializes the database and observability platforms, starts the HTTP server,+// and handles graceful shutdown.+func main() {+ figure.NewColorFigure("DOKU Ingester", "", "yellow", true).Print()+ // Configure global settings for the zerolog logger+ zerolog.TimeFieldFormat = zerolog.TimeFormatUnix+ log.Info().Msg("Starting Doku Ingester")++ // Use flag package to parse the configuration file+ configFilePath := flag.String("config", "./config.yml", "Path to the Doku Ingester config file")+ flag.Parse()++ // Load the configuration+ cfg, err := config.LoadConfiguration(*configFilePath)+ if err != nil {+ log.Fatal().Err(err).Msg("Failed to load configuration file")+ }++ // Load pricing information, either from a local file or from a URL+ if cfg.PricingInfo.URL != "" {+ log.Info().Msgf("Initializing LLM Pricing Information from URL '%s'", cfg.PricingInfo.URL)+ err = cost.LoadPricing("", cfg.PricingInfo.URL)+ } else if cfg.PricingInfo.LocalFile.Path != "" {+ log.Info().Msgf("Initializing LLM Pricing Information from local file '%s", cfg.PricingInfo.LocalFile.Path)+ err = cost.LoadPricing(cfg.PricingInfo.LocalFile.Path, "")+ }+ if err != nil {+ log.Fatal().Err(err).Msg("Failed to load LLM pricing information")+ }+ log.Info().Msg("Successfully initialized LLM pricing information")++ // Initialize the backend database connection with loaded configuration+ log.Info().Msg("Initializing connection to the backend database")+ err = db.Init(*cfg)+ if err != nil {+ log.Fatal().Msg("Unable to initialize connection to the backend database")+ }+ log.Info().Msg("Successfully initialized connection to the backend database")++ // Initialize observability platform if configured+ if cfg.ObservabilityPlatform.Enabled == true {+ log.Info().Msg("Initializing for your Observability Platform")+ err := obsPlatform.Init(*cfg)+ if err != nil {+ log.Fatal().Msg("Exiting due to error in initializing for your Observability Platform")+ }+ log.Info().Msgf("Setup complete for sending data to %s", obsPlatform.ObservabilityPlatform)+ }++ // Cache eviction setup for the authentication process+ auth.InitializeCacheEviction()++ // Initialize the HTTP server routing+ r := mux.NewRouter()+ r.HandleFunc("/api/push", api.DataHandler).Methods("POST")+ r.HandleFunc("/api/keys", api.APIKeyHandler).Methods("GET", "POST", "DELETE")+ r.HandleFunc("/", api.BaseEndpoint).Methods("GET")++ // Define and start the HTTP server+ server := &http.Server{+ Addr: ":" + cfg.IngesterPort,+ Handler: r,+ ReadTimeout: 10 * time.Second,+ WriteTimeout: 10 * time.Second,+ IdleTimeout: 30 * time.Second,+ }++ // Starts the HTTP server in a goroutine and logs any error upon starting.+ go func() {+ log.Info().Msg("Server listening on port " + cfg.IngesterPort)+ if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {+ log.Fatal().Err(err).Msg("Could not listen on port " + cfg.IngesterPort)+ }+ }()++ waitForShutdown(server)+}diff --git a/src/ingester/obsPlatform/datadog.go b/src/ingester/obsPlatform/datadog.gonew file mode 100644index 00000000..c1b0e709--- /dev/null+++ b/src/ingester/obsPlatform/datadog.go@@ -0,0 +1,335 @@+package obsPlatform++import (+ "bytes"+ "fmt"+ "net/http"+ "strings"+ "time"++ "github.com/rs/zerolog/log"+)++func configureDataDogData(data map[string]interface{}) {+ currentTime := time.Now().Unix()++ if data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions" || data["endpoint"] == "cohere.generate" || data["endpoint"] == "cohere.chat" || data["endpoint"] == "cohere.summarize" || data["endpoint"] == "anthropic.completions" {+ if data["finishReason"] == nil {+ data["finishReason"] = "null"+ }+ // Create individual metric strings+ metricStrings := []string{+ fmt.Sprintf(`{+ "metric": "doku.llm.completion.tokens",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]+ }`, currentTime, data["completionTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.prompt.tokens",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]+ }`, currentTime, data["promptTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.total.tokens",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]+ }`, currentTime, data["totalTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.request.duration",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.usage.cost",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finishReason:%v"]+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ }++ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")+ }++ logEntries := []string{+ fmt.Sprintf(`{+ "ddsource": "doku",+ "message": "%s",+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",+ "hostname": "doku-ingester",+ "service": "%v"+ }`, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),+ fmt.Sprintf(`{+ "ddsource": "doku",+ "message": "%s",+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:response",+ "hostname": "doku-ingester",+ "service": "%v"+ }`, normalizeString(data["response"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),+ }++ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")+ }+ } else if data["endpoint"] == "openai.embeddings" || data["endpoint"] == "cohere.embed" {+ if data["endpoint"] == "openai.embeddings" {++ metricStrings := []string{+ fmt.Sprintf(`{+ "metric": "doku.llm.prompt.tokens",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]+ }`, currentTime, data["promptTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.total.tokens",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]+ }`, currentTime, data["totalTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.request.duration",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.usage.cost",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ }++ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")+ }++ logEntries := []string{+ fmt.Sprintf(`{+ "ddsource": "doku",+ "message": "%s",+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",+ "hostname": "doku-ingester",+ "service": "%v"+ }`, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),+ }++ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")+ }+ } else {+ metricStrings := []string{+ fmt.Sprintf(`{+ "metric": "doku.llm.prompt.tokens",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]+ }`, currentTime, data["promptTokens"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.request.duration",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.usage.cost",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v"]+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ }++ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")+ }++ logEntries := []string{+ fmt.Sprintf(`{+ "ddsource": "doku",+ "message": "%s",+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",+ "hostname": "doku-ingester",+ "service": "%v"+ }`, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),+ }++ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")+ }+ }+ } else if data["endpoint"] == "openai.fine_tuning" {+ metricStrings := []string{+ fmt.Sprintf(`{+ "metric": "doku.llm.request.duration",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "finetuneJobId:%v"]+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finetuneJobId"]),+ }++ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")+ }+ } else if data["endpoint"] == "openai.images.create" || data["endpoint"] == "openai.images.create.variations" {+ metricStrings := []string{+ fmt.Sprintf(`{+ "metric": "doku.llm.request.duration",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "imageSize:%v","imageQuality:%v"]+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.usage.cost",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "imageSize:%v","imageQuality:%v"]+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"]),+ }++ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")+ }++ var logEntries []string+ // Check the condition for the prompt+ if data["endpoint"] != "openai.images.create.variations" {+ var promptMessage string+ if data["model"] == "dall-e-2" {+ // Assuming data["prompt"] exists and is a string+ promptMessage = normalizeString(data["prompt"].(string))+ } else {+ // Assuming data["revisedPrompt"] exists and is a string+ promptMessage = normalizeString(data["revisedPrompt"].(string))+ }++ // Build the prompt log+ logEntries = append(logEntries, fmt.Sprintf(`{+ "ddsource": "doku",+ "message": "%s",+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",+ "hostname": "doku-ingester",+ "service": "%v"+ }`, promptMessage, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),+ )+ }++ logEntries = append(logEntries, fmt.Sprintf(`{+ "ddsource": "doku",+ "message": "%s",+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:image",+ "hostname": "doku-ingester",+ "service": "%v"+ }`, data["image"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),+ )++ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")+ }+ } else if data["endpoint"] == "openai.audio.speech.create" {+ metricStrings := []string{+ fmt.Sprintf(`{+ "metric": "doku.llm.request.duration",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "audioVoice:%v"]+ }`, currentTime, data["requestDuration"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"]),+ fmt.Sprintf(`{+ "metric": "doku.llm.usage.cost",+ "type": 0,+ "points": [{ "timestamp": %d, "value": %f }],+ "resources": [{ "name": "doku-ingester", "type": "host" }],+ "tags": ["environment:%v", "endpoint:%v", "applicationName:%v", "source:%v", "model:%v", "audioVoice:%v"]+ }`, currentTime, data["usageCost"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"]),+ }++ metrics := fmt.Sprintf(`{"series": [%s]}`, strings.Join(metricStrings, ","))+ err := sendTelemetryDataDog(metrics, dataDogAPIKey, dataDogMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to DataDog")+ }++ logEntries := []string{+ fmt.Sprintf(`{+ "ddsource": "doku",+ "message": "%s",+ "ddtags": "environment:%v,endpoint:%v,applicationName:%v,source:%v,model:%v,type:prompt",+ "hostname": "doku-ingester",+ "service": "%v"+ }`, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["applicationName"]),+ }++ logs := fmt.Sprintf("[%s]", strings.Join(logEntries, ","))+ err = sendTelemetryDataDog(logs, dataDogAPIKey, dataDogLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to DataDog")+ }+ }+}++func sendTelemetryDataDog(telemetryData, headerKey string, url string, requestType string) error {+ // Create a new request using http+ req, err := http.NewRequest(requestType, url, bytes.NewBuffer([]byte(telemetryData)))+ if err != nil {+ return fmt.Errorf("Error creating request")+ }++ // Add headers to the request+ req.Header.Set("Accept", "application/json")+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("DD-API-KEY", headerKey)++ // Send the request via a client+ httpClient := &http.Client{}+ resp, err := httpClient.Do(req)+ if err != nil {+ return fmt.Errorf("Error sending request to %v", url)+ } else if resp.StatusCode == 404 {+ return fmt.Errorf("Provided URL %v is not valid", url)+ } else if resp.StatusCode == 403 {+ return fmt.Errorf("Provided credentials are not valid")+ }++ defer resp.Body.Close()++ log.Info().Msgf("Successfully exported data to %v", url)+ return nil+}diff --git a/src/ingester/obsPlatform/newrelic.go b/src/ingester/obsPlatform/newrelic.gonew file mode 100644index 00000000..2b0499ea--- /dev/null+++ b/src/ingester/obsPlatform/newrelic.go@@ -0,0 +1,416 @@+package obsPlatform++import (+ "bytes"+ "fmt"+ "net/http"+ "strconv"+ "strings"+ "time"++ "github.com/rs/zerolog/log"+)++func configureNewRelicData(data map[string]interface{}) {+ // The current time for the timestamp field.+ currentTime := strconv.FormatInt(time.Now().Unix(), 10)++ if data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions" || data["endpoint"] == "cohere.generate" || data["endpoint"] == "cohere.chat" || data["endpoint"] == "cohere.summarize" || data["endpoint"] == "anthropic.completions" {+ if data["finishReason"] == nil {+ data["finishReason"] = "null"+ }++ jsonMetrics := []string{+ fmt.Sprintf(`{+ "name": "doku.LLM.Completion.Tokens",+ "type": "gauge",+ "value": %f,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}+ }`, data["completionTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Prompt.Tokens",+ "type": "gauge",+ "value": %f,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}+ }`, data["promptTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Total.Tokens",+ "type": "gauge",+ "value": %f,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}+ }`, data["totalTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Request.Duration",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Usage.Cost",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finishReason": "%v"}+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"]),+ }++ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))++ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")+ }++ // Use the values from the provided log lines and adapt them to the desired format.+ logs := []string{+ fmt.Sprintf(`{+ "timestamp": %s,+ "message": "%s",+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "type": "response"+ }+ }`, currentTime, normalizeString(data["response"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]), // Assuming 'response' is present in data and is a string.++ fmt.Sprintf(`{+ "timestamp": %s,+ "message": "%s",+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "type": "prompt"+ }+ }`, currentTime, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]), // Assuming 'prompt' is present in data and is a string.+ }++ // Combine the individual log entries into a full JSON payload+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))++ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")+ }++ } else if data["endpoint"] == "openai.embeddings" || data["endpoint"] == "cohere.embed" {+ if data["endpoint"] == "openai.embeddings" {+ jsonMetrics := []string{+ fmt.Sprintf(`{+ "name": "doku.LLM.Prompt.Tokens",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": { "environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}+ }`, data["promptTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Total.Tokens",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}+ }`, data["totalTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Request.Duration",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": { "environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Usage.Cost",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": { "environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ }++ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))++ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")+ }++ // Build log entries with varying labels+ logs := []string{+ fmt.Sprintf(`{+ "timestamp": %s,+ "message": "%s",+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "type": "prompt"+ }+ }`, currentTime, normalizeString(data["prompt"].(string)), data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ }+ // Combine the individual log entries into a full JSON payload+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))++ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")+ }+ } else {+ jsonMetrics := []string{+ fmt.Sprintf(`{+ "name": "doku.LLM.Prompt.Tokens",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}+ }`, data["promptTokens"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Request.Duration",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Usage.Cost",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v"}+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ }++ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))++ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Merics to New Relic")+ }++ logs := []string{+ // Assuming 'prompt' exists and is a string in data+ fmt.Sprintf(`{+ "timestamp": "%s",+ "message": "%s",+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "type": "prompt"+ }+ }`,+ currentTime,+ data["prompt"],+ data["environment"],+ data["endpoint"],+ data["applicationName"],+ data["sourceLanguage"],+ data["model"]),+ }++ // Combine the individual log entries into a full JSON payload+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))++ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")+ }++ }+ } else if data["endpoint"] == "openai.fine_tuning" {+ jsonMetrics := []string{+ fmt.Sprintf(`{+ "name": "doku.LLM.Request.Duration",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "finetuneJobId": "%v"}+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finetuneJobId"]),+ }+ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))++ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")+ }+ } else if data["endpoint"] == "openai.images.create" || data["endpoint"] == "openai.images.create.variations" {+ jsonMetrics := []string{+ fmt.Sprintf(`{+ "name": "doku.LLM.Request.Duration",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "imageSize": "%v",+ "imageQuality": "%v"+ }+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"]),+ fmt.Sprintf(`{+ "name": "doku.LLM.Usage.Cost",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "imageSize": "%v",+ "imageQuality": "%v"+ }+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"]),+ }++ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))++ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")+ }++ var logs []string+ // Check the condition for the prompt+ if data["endpoint"] != "openai.images.create.variations" {+ var promptMessage string+ if data["model"] == "dall-e-2" {+ // Assuming data["prompt"] exists and is a string+ promptMessage = normalizeString(data["prompt"].(string))+ } else {+ // Assuming data["revisedPrompt"] exists and is a string+ promptMessage = normalizeString(data["revisedPrompt"].(string))+ }++ // Build the prompt log+ logs = append(logs, fmt.Sprintf(`{+ "timestamp": "%s",+ "message": "%s",+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "type": "prompt"+ }+ }`, currentTime, promptMessage, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]))+ }++ // Build the image log+ logs = append(logs, fmt.Sprintf(`{+ "timestamp": "%s",+ "message": "%s",+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "type": "image"+ }+ }`, currentTime, data["image"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]))++ // Combine the individual log entries into a full JSON payload+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))++ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")+ }++ } else if data["endpoint"] == "openai.audio.speech.create" {+ jsonMetrics := []string{+ fmt.Sprintf(`{+ "name": "doku_llm.RequestDuration",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "audioVoice": "%v"}+ }`, data["requestDuration"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"]),+ fmt.Sprintf(`{+ "name": "doku_llm.UsageCost",+ "type": "gauge",+ "value": %v,+ "timestamp": %s,+ "attributes": {"environment": "%v", "endpoint": "%v", "applicationName": "%v", "source": "%v", "model": "%v", "audioVoice": "%v"}+ }`, data["usageCost"], currentTime, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"]),+ }++ // Join the individual metric strings into a comma-separated string and enclose in a JSON array.+ jsonData := fmt.Sprintf(`[{"metrics": [%s]}]`, strings.Join(jsonMetrics, ","))++ err := sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicMetricsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Metrics to New Relic")+ }++ logs := []string{+ // Assuming 'prompt' exists and is a string in data+ fmt.Sprintf(`{+ "timestamp": "%s",+ "message": "%s",+ "attributes": {+ "environment": "%v",+ "endpoint": "%v",+ "applicationName": "%v",+ "source": "%v",+ "model": "%v",+ "type": "prompt"+ }+ }`, currentTime, data["prompt"], data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"]),+ }++ // Combine the individual log entries into a full JSON payload+ jsonData = fmt.Sprintf(`[{"logs": [%s]}]`, strings.Join(logs, ","))++ err = sendTelemetryNewRelic(jsonData, newRelicLicenseKey, "Api-Key", newRelicLogsUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending Logs to New Relic")+ }+ }+}++func sendTelemetryNewRelic(telemetryData, authHeader string, headerKey string, url string, requestType string) error {++ req, err := http.NewRequest(requestType, url, bytes.NewBufferString(telemetryData))+ if err != nil {+ return fmt.Errorf("Error creating request")+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set(fmt.Sprintf("%s", headerKey), authHeader)++ resp, err := httpClient.Do(req)+ if err != nil {+ return fmt.Errorf("Error sending request to %v", url)+ } else if resp.StatusCode == 404 {+ return fmt.Errorf("Provided URL %v is not valid", url)+ } else if resp.StatusCode == 403 {+ return fmt.Errorf("Provided credentials are not valid")+ }++ defer resp.Body.Close()++ log.Info().Msgf("Successfully exported data to %v", url)+ return nil+}diff --git a/src/ingester/obsPlatform/obsPlatform.go b/src/ingester/obsPlatform/obsPlatform.gonew file mode 100644index 00000000..6b76074e--- /dev/null+++ b/src/ingester/obsPlatform/obsPlatform.go@@ -0,0 +1,237 @@+package obsPlatform++import (+ "bytes"+ "fmt"+ "ingester/config"+ "net/http"+ "regexp"+ "strconv"+ "strings"+ "time"++ "github.com/rs/zerolog/log"+)++var (+ ObservabilityPlatform string // ObservabilityPlatform contains the information on the platform in use.+ httpClient *http.Client // httpClient is the HTTP client used to send data to the Observability Platform.+ grafanaPromUrl string // grafanaPrometheusUrl is the URL used to send data to Grafana Prometheus.+ grafanaPromUsername string // grafanaPrometheusUsername is the username used to send data to Grafana Prometheus.+ grafanaLokiUrl string // grafanaPostUrl is the URL used to send data to Grafana Loki.+ grafanaLokiUsername string // grafanaLokiUsername is the username used to send data to Grafana Loki.+ grafanaAccessToken string // grafanaAccessToken is the access token used to send data to Grafana.+ newRelicLicenseKey string // newRelicKey is the key used to send data to New Relic.+ newRelicMetricsUrl string // newRelicMetricsUrl is the URL used to send data to New Relic.+ newRelicLogsUrl string // newRelicLogsUrl is the URL used to send logs to New Relic.+ dataDogMetricsUrl string // dataDogMetricsUrl is the URL used to send data to DataDog.+ dataDogLogsUrl string // dataDogLogsUrl is the URL used to send logs to DataDog.+ dataDogAPIKey string // dataDogAPIKey is the API key used to send data to DataDog.+)++func normalizeString(s string) string {+ // Remove backslashes+ s = strings.ReplaceAll(s, `\`, "")++ // Replace double quotes with single quotes+ s = strings.ReplaceAll(s, `"`, "'")++ // Normalize spacing around hyphens: no space before, one space after hyphens+ s = strings.ReplaceAll(s, " - ", " -")+ s = strings.ReplaceAll(s, " -", " -")+ s = strings.ReplaceAll(s, "- ", " - ")++ // Collapse multiple spaces into a single space+ re := regexp.MustCompile(`\s+`)+ s = re.ReplaceAllString(s, " ")++ // Trim leading and trailing whitespace+ s = strings.TrimSpace(s)++ return s+}++func Init(cfg config.Configuration) error {+ httpClient = &http.Client{Timeout: 5 * time.Second}+ if cfg.ObservabilityPlatform.GrafanaCloud.LokiURL != "" {+ grafanaPromUrl = cfg.ObservabilityPlatform.GrafanaCloud.PromURL+ grafanaPromUsername = cfg.ObservabilityPlatform.GrafanaCloud.PromUsername+ grafanaLokiUrl = cfg.ObservabilityPlatform.GrafanaCloud.LokiURL+ grafanaLokiUsername = cfg.ObservabilityPlatform.GrafanaCloud.LokiUsername+ grafanaAccessToken = cfg.ObservabilityPlatform.GrafanaCloud.AccessToken+ } else if cfg.ObservabilityPlatform.NewRelic.Key != "" {+ newRelicLicenseKey = cfg.ObservabilityPlatform.NewRelic.Key+ newRelicMetricsUrl = cfg.ObservabilityPlatform.NewRelic.MetricsURL+ newRelicLogsUrl = cfg.ObservabilityPlatform.NewRelic.LogsURL+ } else if cfg.ObservabilityPlatform.DataDog.APIKey != "" {+ dataDogMetricsUrl = cfg.ObservabilityPlatform.DataDog.MetricsURL+ dataDogLogsUrl = cfg.ObservabilityPlatform.DataDog.LogsURL+ dataDogAPIKey = cfg.ObservabilityPlatform.DataDog.APIKey+ }+ return nil+}++// SendToPlatform sends observability data to the appropriate platform.+func SendToPlatform(data map[string]interface{}) {+ if grafanaLokiUrl != "" {+ if data["endpoint"] == "openai.chat.completions" || data["endpoint"] == "openai.completions" || data["endpoint"] == "cohere.generate" || data["endpoint"] == "cohere.chat" || data["endpoint"] == "cohere.summarize" || data["endpoint"] == "anthropic.completions" {+ if data["finishReason"] == nil {+ data["finishReason"] = "null"+ }+ metrics := []string{+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v completionTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["completionTokens"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v promptTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["promptTokens"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v totalTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["totalTokens"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["requestDuration"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finishReason=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finishReason"], data["usageCost"]),+ }+ var metricsBody = []byte(strings.Join(metrics, "\n"))+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")+ }++ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)++ response_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"response\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), normalizeString(data["response"].(string))))+ err = sendTelemetry(response_log, authHeader, grafanaLokiUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")+ }++ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), normalizeString(data["prompt"].(string))))+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")+ }+ } else if data["endpoint"] == "openai.embeddings" || data["endpoint"] == "cohere.embed" {+ if data["endpoint"] == "openai.embeddings" {+ metrics := []string{+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v promptTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["promptTokens"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v totalTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["totalTokens"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["requestDuration"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["usageCost"]),+ }+ var metricsBody = []byte(strings.Join(metrics, "\n"))+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")+ }++ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["prompt"]))+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")+ }+ } else {+ metrics := []string{+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v promptTokens=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["promptTokens"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["requestDuration"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["usageCost"]),+ }+ var metricsBody = []byte(strings.Join(metrics, "\n"))+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")+ }++ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["prompt"]))+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")+ }+ }+ } else if data["endpoint"] == "openai.fine_tuning" {+ metrics := []string{+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,finetuneJobId=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["finetuneJobId"], data["requestDuration"]),+ }+ var metricsBody = []byte(strings.Join(metrics, "\n"))+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")+ }+ } else if data["endpoint"] == "openai.images.create" || data["endpoint"] == "openai.images.create.variations" {+ metrics := []string{+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,imageSize=%v,imageQuality=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"], data["requestDuration"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,imageSize=%v,imageQuality=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["imageSize"], data["imageQuality"], data["usageCost"]),+ }+ var metricsBody = []byte(strings.Join(metrics, "\n"))+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")+ }++ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)+ if data["endpoint"] != "openai.images.create.variations" {+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["revisedPrompt"]))+ if data["model"] == "dall-e-2" {+ prompt_log = []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["prompt"]))+ }+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")+ }+ }+ image_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"image\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["image"]))+ err = sendTelemetry(image_log, authHeader, grafanaLokiUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")+ }+ } else if data["endpoint"] == "openai.audio.speech.create" {+ metrics := []string{+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,audioVoice=%v requestDuration=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"], data["requestDuration"]),+ fmt.Sprintf(`doku_llm,environment=%v,endpoint=%v,applicationName=%v,source=%v,model=%v,audioVoice=%v usageCost=%v`, data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], data["audioVoice"], data["usageCost"]),+ }+ var metricsBody = []byte(strings.Join(metrics, "\n"))+ authHeader := fmt.Sprintf("Bearer %v:%v", grafanaPromUsername, grafanaAccessToken)+ err := sendTelemetry(metricsBody, authHeader, grafanaPromUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Prometheus")+ }++ authHeader = fmt.Sprintf("Bearer %v:%v", grafanaLokiUsername, grafanaAccessToken)+ prompt_log := []byte(fmt.Sprintf("{\"streams\": [{\"stream\": {\"environment\": \"%v\",\"endpoint\": \"%v\", \"applicationName\": \"%v\", \"source\": \"%v\", \"model\": \"%v\", \"type\": \"prompt\" }, \"values\": [[\"%s\", \"%v\"]]}]}", data["environment"], data["endpoint"], data["applicationName"], data["sourceLanguage"], data["model"], strconv.FormatInt(time.Now().UnixNano(), 10), data["prompt"]))+ err = sendTelemetry(prompt_log, authHeader, grafanaLokiUrl, "POST")+ if err != nil {+ log.Error().Err(err).Msgf("Error sending data to Grafana Cloud Loki")+ }+ }+ } else if newRelicMetricsUrl != "" {+ configureNewRelicData(data)+ } else if dataDogMetricsUrl != "" {+ configureDataDogData(data)+ } else {+ log.Info().Msg("No Observability Platform configured")+ }+}++func sendTelemetry(telemetryData []byte, authHeader string, url string, requestType string) error {++ req, err := http.NewRequest(requestType, url, bytes.NewBuffer(telemetryData))+ if err != nil {+ return fmt.Errorf("Error creating request")+ }+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Authorization", authHeader)++ resp, err := httpClient.Do(req)+ if err != nil {+ return fmt.Errorf("Error sending request to %v", url)+ } else if resp.StatusCode == 404 {+ return fmt.Errorf("Provided URL %v is not valid", url)+ } else if resp.StatusCode == 401 {+ return fmt.Errorf("Provided credentials are not valid")+ }++ defer resp.Body.Close()++ log.Info().Msgf("Successfully exported data to %v", url)+ return nil+}