RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(common): move IngestionLog and ingest-log sink out of stub.go (#22006)
xugangqiangb7a9a61add4a
diff --git a/internal/common/ingest_log.go b/internal/common/ingest_log.gonew file mode 100644index 0000000000..e95505b099--- /dev/null+++ b/internal/common/ingest_log.go@@ -0,0 +1,54 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package common++import (+ "time"+)++// IngestionLog is the structured record for ingestion (document-parse) lifecycle+// events. It is a shared type used by both the OSS and EE builds; the sink+// implementation (LogIngest and friends) is provided per edition (the OSS build+// keeps a no-op seam in stub.go, the EE build provides the real sink).+type IngestionLog struct {+ EventTime time.Time `json:"event_time"`+ TenantID string `json:"tenant_id"`+ DatasetID string `json:"dataset_id"`+ DocID string `json:"doc_id"`+ UserID string `json:"user_id"`+ UserName string `json:"user_name,omitempty"`+ TenantName string `json:"tenant_name,omitempty"`+ Worker string `json:"worker"`+ TaskID string `json:"task_id"`+ EventType string `json:"event_type"`+ Status string `json:"status"`+ ErrorInfo string `json:"error_info,omitempty"`+ Filename string `json:"filename"`+ FileType string `json:"file_type"`+ FileSize int64 `json:"file_size"`+ ParserID string `json:"parser_id"`+ UploadTime time.Time `json:"upload_time"`+ EnqueueTime time.Time `json:"enqueue_time"`+ BeginTime time.Time `json:"begin_time"`+ EndTime time.Time `json:"end_time"`+ DurationMS int64 `json:"duration_ms"`+ WaitDurationS float64 `json:"wait_duration_s"`+ ParseDurationS float64 `json:"parse_duration_s"`+ Progress float64 `json:"progress"`+ RunCount int `json:"run_count"`+ Code int `json:"code"`+ Operation map[string]any `json:"operation,omitempty"`+}diff --git a/internal/common/ingest_log_test.go b/internal/common/ingest_log_test.gonew file mode 100644index 0000000000..9eb98bb8df--- /dev/null+++ b/internal/common/ingest_log_test.go@@ -0,0 +1,111 @@+//go:build !ee++package common++import (+ "encoding/json"+ "testing"+)++// TestIngestionLogJSONContract pins the JSON shape of the OSS IngestionLog type+// so a future refactor (e.g. moving it out of stub.go into its own file, or+// changing field tags) is caught if it alters the wire format that the EE sink+// and downstream consumers rely on.+func TestIngestionLogJSONContract(t *testing.T) {+ log := IngestionLog{+ TenantID: "t1",+ DatasetID: "d1",+ DocID: "doc1",+ UserID: "u1",+ TaskID: "task1",+ EventType: "completed",+ Status: "success",+ Filename: "a.pdf",+ FileType: "pdf",+ FileSize: 1024,+ ParserID: "parser1",+ Progress: 1.0,+ RunCount: 1,+ Code: 0,+ }++ b, err := json.Marshal(log)+ if err != nil {+ t.Fatalf("marshal IngestionLog: %v", err)+ }+ var m map[string]any+ if err := json.Unmarshal(b, &m); err != nil {+ t.Fatalf("unmarshal: %v", err)+ }++ // Required, always-present keys.+ for _, key := range []string{+ "tenant_id", "dataset_id", "doc_id", "user_id", "task_id",+ "event_type", "status", "filename", "file_type", "file_size",+ "parser_id", "progress", "run_count", "code",+ } {+ if _, ok := m[key]; !ok {+ t.Errorf("expected key %q present in JSON, got %s", key, string(b))+ }+ }++ // Optional fields must be omitted when empty (omitempty contract).+ for _, key := range []string{"user_name", "tenant_name", "error_info", "operation"} {+ if _, ok := m[key]; ok {+ t.Errorf("expected optional key %q omitted when empty, got %s", key, string(b))+ }+ }+}++// TestIngestionLogOptionalFieldsEmitted verifies that non-empty optional fields+// are serialized (the omitempty contract works both ways).+func TestIngestionLogOptionalFieldsEmitted(t *testing.T) {+ log := IngestionLog{+ TaskID: "task1",+ UserName: "alice",+ TenantName: "acme",+ ErrorInfo: "boom",+ Operation: map[string]any{"step": "parse"},+ }+ b, err := json.Marshal(log)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ var m map[string]any+ if err := json.Unmarshal(b, &m); err != nil {+ t.Fatalf("unmarshal: %v", err)+ }+ for _, key := range []string{"user_name", "tenant_name", "error_info", "operation"} {+ if _, ok := m[key]; !ok {+ t.Errorf("expected optional key %q emitted, got %s", key, string(b))+ }+ }+}++// TestIngestLogSinkNoop pins the OSS no-op behavior of the ingest-log sink:+// InitIngestLog returns nil, LogIngest is callable without panicking, and+// SyncIngestLog is callable without panicking. The real sink lives in the EE+// build; the OSS binary must keep the seam compiling and running safely.+func TestIngestLogSinkNoop(t *testing.T) {+ if err := InitIngestLog("file", FileOutput{}, "svc"); err != nil {+ t.Fatalf("InitIngestLog returned error in OSS build: %v", err)+ }++ func() {+ defer func() {+ if r := recover(); r != nil {+ t.Fatalf("LogIngest panicked: %v", r)+ }+ }()+ LogIngest(&IngestionLog{TaskID: "t", EventType: "completed"})+ }()++ func() {+ defer func() {+ if r := recover(); r != nil {+ t.Fatalf("SyncIngestLog panicked: %v", r)+ }+ }()+ SyncIngestLog()+ }()+}diff --git a/internal/common/stub.go b/internal/common/stub.goindex a8b2eb9f40..6f8e64e88f 100644--- a/internal/common/stub.go+++ b/internal/common/stub.go@@ -68,40 +68,12 @@ func InitIngestLog(logType string, file FileOutput, serviceName string) error {return nil}-type IngestionLog struct {- EventTime time.Time `json:"event_time"`- TenantID string `json:"tenant_id"`- DatasetID string `json:"dataset_id"`- DocID string `json:"doc_id"`- UserID string `json:"user_id"`- UserName string `json:"user_name,omitempty"`- TenantName string `json:"tenant_name,omitempty"`- Worker string `json:"worker"`- TaskID string `json:"task_id"`- EventType string `json:"event_type"`- Status string `json:"status"`- ErrorInfo string `json:"error_info,omitempty"`- Filename string `json:"filename"`- FileType string `json:"file_type"`- FileSize int64 `json:"file_size"`- ParserID string `json:"parser_id"`- UploadTime time.Time `json:"upload_time"`- EnqueueTime time.Time `json:"enqueue_time"`- BeginTime time.Time `json:"begin_time"`- EndTime time.Time `json:"end_time"`- DurationMS int64 `json:"duration_ms"`- WaitDurationS float64 `json:"wait_duration_s"`- ParseDurationS float64 `json:"parse_duration_s"`- Progress float64 `json:"progress"`- RunCount int `json:"run_count"`- Code int `json:"code"`- Operation map[string]any `json:"operation,omitempty"`-}-+// SyncIngestLog is the OSS no-op flush for the ingest logger. The real sink lives+// in the EE build; the OSS binary keeps only the instrumentation seam so the+// server entrypoint's deferred flush (cmd/ragflow_server.go) compiles and runs.func SyncIngestLog() {-}-// LogIngest is the shared upstream seam. In the OSS build it is a no-op; the EE+// LogIngest is the OSS no-op implementation of the shared upstream seam. The EE// build provides the real sink.func LogIngest(log *IngestionLog, fields ...zap.Field) {}