Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(cmd): make pprof startup testable and split unit/integration tiers (#21491)

xugangqiang3eb36c449b53
diff --git a/cmd/ragflow_server.go b/cmd/ragflow_server.go
index 994e5ba152..ace822d810 100644
--- a/cmd/ragflow_server.go
+++ b/cmd/ragflow_server.go
@@ -22,6 +22,7 @@ import (
"flag"
"fmt"
"io"
+ "net"
"net/http"
_ "net/http/pprof"
"os"
@@ -58,19 +59,38 @@ func isStandaloneDeepDocServer(st common.ServerType) bool {
st == common.ServerTypeDeepDocAscend
}
-// startPProfIfRequested launches the pprof HTTP server on localhost:6060.
-// The blank "net/http/pprof" import above registers the handlers on
-// http.DefaultServeMux, so only the listener needs starting. Both boot
-// paths (standalone DeepDoc and the regular one) call it right after the
-// logger is up, so its startup line lands in the run's log file. A bind
-// failure is logged and non-fatal, matching the previous inline behavior.
-func startPProfIfRequested() {
+// ErrPProfBindFailed is returned by startPProf when the profiling port is
+// unavailable. Callers treat it as non-fatal: the inference server matters more
+// than profiling, so a bind failure is logged and startup continues.
+var ErrPProfBindFailed = errors.New("pprof bind failed")
+
+// startPProf opens a TCP listener on addr and serves the pprof handlers
+// (registered by the blank "net/http/pprof" import) for the process lifetime.
+// It returns ErrPProfBindFailed if addr is already taken instead of crashing
+// the server. The address is configurable via pprofAddr so tests (and ops) can
+// avoid the hard-coded localhost:6060 clash.
+func startPProf(addr string) error {
+ ln, err := net.Listen("tcp", addr)
+ if err != nil {
+ return fmt.Errorf("%w: %v", ErrPProfBindFailed, err)
+ }
go func() {
- common.Info("Starting pprof server", zap.String("addr", "localhost:6060"))
- if pprofErr := http.ListenAndServe("localhost:6060", nil); pprofErr != nil {
+ common.Info("Starting pprof server", zap.String("addr", addr))
+ if pprofErr := http.Serve(ln, nil); pprofErr != nil {
common.Error("pprof server failed", pprofErr)
}
}()
+ return nil
+}
+
+// pprofAddr returns the listen address for the pprof server. It honors
+// RAGFLOW_PPROF_ADDR so callers/tests can override the default localhost:6060,
+// eliminating hard-coded-port clashes between concurrent tests and processes.
+func pprofAddr() string {
+ if a := os.Getenv("RAGFLOW_PPROF_ADDR"); a != "" {
+ return a
+ }
+ return "localhost:6060"
}
// parseArgsFrom parses argv (argv[0] is the program name, mirroring os.Args)
@@ -392,7 +412,9 @@ func main() {
// path skips, so --profile must be honored here or the flag the
// --deepdoc help text advertises would be silently ignored.
if arguments.EnablePProf {
- startPProfIfRequested()
+ if err := startPProf(pprofAddr()); err != nil {
+ common.Error("pprof server failed", err)
+ }
}
if err = runDeepDoc(ctx, finalName, arguments); err != nil {
fmt.Printf("Failed to start DEEPDOC: %v\n", err)
@@ -515,7 +537,9 @@ func main() {
// Start pprof server if requested
if arguments.EnablePProf {
- startPProfIfRequested()
+ if err := startPProf(pprofAddr()); err != nil {
+ common.Error("pprof server failed", err)
+ }
}
// Initialize database. The migrate mode stops here: it needs neither the
diff --git a/cmd/ragflow_server_pprof_integration_test.go b/cmd/ragflow_server_pprof_integration_test.go
new file mode 100644
index 0000000000..5d47e7edf3
--- /dev/null
+++ b/cmd/ragflow_server_pprof_integration_test.go
@@ -0,0 +1,144 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+//go:build integration
+
+package main
+
+import (
+ "context"
+ "os"
+ "os/exec"
+ "strings"
+ "testing"
+ "time"
+)
+
+// These integration tests pin the end-to-end contract that main()'s standalone
+// DeepDoc early-return path starts (or, without --profile, does NOT start) the
+// pprof server. They re-execute the test binary as a subprocess (the pattern
+// established by TestMainMigrateReachesConfiguration), but unlike the old
+// design they probe the LIVE child from the parent and then kill it — they do
+// NOT wait for main() to return. That makes them robust to a real DeepDoc
+// server that blocks for the process lifetime (the regression behind the
+// #1036 CI failure, where the OSS runDeepDoc stub returns but the EE one does
+// not). The pprof port is ephemeral (RAGFLOW_PPROF_ADDR), eliminating the
+// hard-coded localhost:6060 clash.
+//
+// When the DeepDoc server cannot actually boot in the test environment (e.g.
+// EE models absent on a runner), the child process exits before the probe
+// succeeds; the tests Skip rather than fail, because the unit TestStartPProf*
+// tests already cover the pprof contract in isolation.
+//
+// The two helper functions below live in the non-tagged pprof test file so they
+// are compiled into every build tier.
+
+const pprofChildEnv = "RAGFLOW_PPROF_CHILD"
+
+// runPProfChild re-executes this test binary as a subprocess with the child
+// branch enabled and an ephemeral RAGFLOW_PPROF_ADDR, returning the running
+// command and a channel closed when the child process exits. Any inherited
+// RAGFLOW_PPROF_ADDR / child-env value is stripped so the test's override wins
+// (os.Getenv returns the first match).
+func runPProfChild(t *testing.T, addr string) (*exec.Cmd, <-chan struct{}) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+ t.Cleanup(cancel)
+
+ env := make([]string, 0, len(os.Environ())+2)
+ for _, e := range os.Environ() {
+ if strings.HasPrefix(e, "RAGFLOW_PPROF_ADDR=") || strings.HasPrefix(e, pprofChildEnv+"=") {
+ continue
+ }
+ env = append(env, e)
+ }
+ env = append(env, pprofChildEnv+"=1", "RAGFLOW_PPROF_ADDR="+addr)
+
+ cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^"+t.Name()+"$", "-test.v")
+ cmd.Env = env
+ if err := cmd.Start(); err != nil {
+ t.Fatalf("start child: %v", err)
+ }
+ exited := make(chan struct{})
+ go func() {
+ _ = cmd.Wait()
+ close(exited)
+ }()
+ return cmd, exited
+}
+
+// TestDeepDocEarlyPathWiresPProf verifies that `--deepdoc --profile` actually
+// starts pprof on the configured address — the regression guard for the early
+// return path silently dropping the flag.
+func TestDeepDocEarlyPathWiresPProf(t *testing.T) {
+ if os.Getenv(pprofChildEnv) == "1" {
+ os.Args = []string{os.Args[0], "--deepdoc", "--profile"}
+ pprofChildBoot()
+ select {} // stay alive; the parent kills us after probing
+ }
+
+ addr := freeLocalAddr(t)
+ cmd, exited := runPProfChild(t, addr)
+ defer cmd.Process.Kill()
+
+ url := "http://" + addr + "/debug/pprof/"
+ deadline := time.Now().Add(15 * time.Second)
+ for time.Now().Before(deadline) {
+ if probeReachable(url) {
+ return
+ }
+ select {
+ case <-exited:
+ t.Skip("deepdoc server could not boot in this environment (e.g. EE models absent); unit TestStartPProf* cover the pprof contract")
+ default:
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ t.Fatalf("pprof endpoint %s never became reachable from --deepdoc --profile", url)
+}
+
+// TestDeepDocEarlyPathNoPProfWithoutFlag is the adversarial guard for the flag
+// gate: `--deepdoc` WITHOUT `--profile` must not open the profiling port.
+func TestDeepDocEarlyPathNoPProfWithoutFlag(t *testing.T) {
+ if os.Getenv(pprofChildEnv) == "1" {
+ os.Args = []string{os.Args[0], "--deepdoc"} // no --profile
+ pprofChildBoot()
+ select {} // stay alive; the parent kills us after probing
+ }
+
+ addr := freeLocalAddr(t)
+ cmd, exited := runPProfChild(t, addr)
+ defer cmd.Process.Kill()
+
+ url := "http://" + addr + "/debug/pprof/"
+ deadline := time.Now().Add(15 * time.Second)
+ reached := false
+ for time.Now().Before(deadline) {
+ if probeReachable(url) {
+ reached = true
+ break
+ }
+ select {
+ case <-exited:
+ t.Skip("deepdoc server could not boot in this environment (e.g. EE models absent); unit TestStartPProf* cover the pprof contract")
+ default:
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ if reached {
+ t.Fatalf("pprof endpoint reachable from --deepdoc WITHOUT --profile (flag gate broken)")
+ }
+}
diff --git a/cmd/ragflow_server_pprof_test.go b/cmd/ragflow_server_pprof_test.go
index 9255069c80..1b060f77c0 100644
--- a/cmd/ragflow_server_pprof_test.go
+++ b/cmd/ragflow_server_pprof_test.go
@@ -17,42 +17,32 @@
package main
import (
- "context"
"errors"
- "fmt"
"net"
"net/http"
- "os"
- "os/exec"
- "strings"
"testing"
"time"
)
-// The DeepDoc early boot returns before the regular pprof block (which sits
-// after the infrastructure init that the early return skips), so the early
-// path must start the profiling endpoint itself. These tests pin that
-// contract end-to-end through main(), re-executing the test binary as a
-// subprocess (the pattern established by TestMainMigrateReachesConfiguration).
-
-const (
- pprofProbeURL = "http://localhost:6060/debug/pprof/"
- pprofSkipExit = 42 // child exit code: port 6060 held by a stray process
- pprofBootWait = 15 * time.Second // max time for main() to finish its early boot
- pprofProbeWait = 15 * time.Second // max time for the pprof endpoint to come up
- pprofSettleWait = 5 * time.Second // silence window for the negative case
-)
-
-// pprofProbeClient dials localhost directly: an ambient HTTP_PROXY must not
-// intercept the probe and fake a negative.
-var pprofProbeClient = &http.Client{
- Timeout: 2 * time.Second,
- Transport: &http.Transport{Proxy: nil},
+// freeLocalAddr returns a localhost TCP address on an OS-assigned ephemeral
+// port. The kernel reserves it until the returned listener is closed, so we
+// only use it to learn a free port and then close it so startPProf can bind it.
+func freeLocalAddr(t *testing.T) string {
+ t.Helper()
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatalf("freeLocalAddr: listen: %v", err)
+ }
+ addr := ln.Addr().String()
+ _ = ln.Close()
+ return addr
}
-// pprofReachable reports whether the pprof index endpoint answers with 200.
-func pprofReachable() bool {
- resp, err := pprofProbeClient.Get(pprofProbeURL)
+// probeReachable reports whether the pprof index endpoint at url answers 200.
+// It dials directly (no proxy) so an ambient HTTP_PROXY cannot fake a negative.
+func probeReachable(url string) bool {
+ client := &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{Proxy: nil}}
+ resp, err := client.Get(url)
if err != nil {
return false
}
@@ -60,10 +50,10 @@ func pprofReachable() bool {
return resp.StatusCode == http.StatusOK
}
-// pprofChildBoot starts main() in the background inside a re-executed test
-// binary and returns a channel closed once main() returns. The OSS runDeepDoc
-// stub returns immediately, but a pprof goroutine started by main() outlives
-// it for the rest of this process, which is what the child assertions observe.
+// pprofChildBoot runs main() in the background and returns a channel closed
+// when main() returns. Used by the integration tier to drive the real server
+// boot path. The OSS runDeepDoc stub returns immediately, but the pprof
+// goroutine outlives it; the integration tests observe that from the parent.
func pprofChildBoot() <-chan struct{} {
done := make(chan struct{})
go func() {
@@ -73,115 +63,56 @@ func pprofChildBoot() <-chan struct{} {
return done
}
-// runPProfChild re-executes this test binary as a subprocess with envVar set
-// so the named test takes its child branch, and maps the exit code:
-// 0 → pass, pprofSkipExit → skip (port 6060 held by a stray process),
-// anything else → fail with the combined child output. It returns the output
-// so callers can add secondary assertions.
-func runPProfChild(t *testing.T, testName, envVar string) string {
- t.Helper()
- ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second)
- defer cancel()
- cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^"+testName+"$", "-test.v")
- cmd.Dir = t.TempDir()
- cmd.Env = append(os.Environ(), envVar+"=1")
- out, err := cmd.CombinedOutput()
- if err != nil {
- var exitErr *exec.ExitError
- if errors.As(err, &exitErr) && exitErr.ExitCode() == pprofSkipExit {
- t.Skip("localhost:6060 is held by another process; cannot test pprof deterministically")
+// TestStartPProfOpensEndpointWhenEnabled pins the happy path: with a free
+// address, startPProf must actually bind and serve /debug/pprof/.
+func TestStartPProfOpensEndpointWhenEnabled(t *testing.T) {
+ addr := freeLocalAddr(t)
+ if err := startPProf(addr); err != nil {
+ t.Fatalf("startPProf(%q) unexpected error: %v", addr, err)
+ }
+ url := "http://" + addr + "/debug/pprof/"
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ if probeReachable(url) {
+ return
}
- t.Fatalf("%s child failed: %v\n%s", testName, err, out)
+ time.Sleep(50 * time.Millisecond)
}
- return string(out)
+ t.Fatalf("pprof endpoint %s not reachable after startPProf", url)
}
-// TestMainDeepDocProfileStartsPProf pins the contract that a standalone
-// DeepDoc server booted with --profile serves pprof on localhost:6060. The
-// DeepDoc early boot returns before the regular pprof block, so without a
-// start in the early path, --deepdoc --profile silently ignores the flag the
-// --deepdoc help text advertises. The secondary assertion on the startup log
-// line proves the endpoint was opened by this boot path, not by a stray
-// listener that slipped past the child's pre-check.
-func TestMainDeepDocProfileStartsPProf(t *testing.T) {
- const envVar = "RAGFLOW_TEST_DEEPDOC_PROFILE"
- if os.Getenv(envVar) == "1" {
- if pprofReachable() {
- os.Exit(pprofSkipExit)
- }
- os.Args = []string{os.Args[0], "--deepdoc", "--profile"}
- pprofChildBoot()
- deadline := time.Now().Add(pprofProbeWait)
- for time.Now().Before(deadline) {
- if pprofReachable() {
- os.Exit(0)
- }
- time.Sleep(50 * time.Millisecond)
- }
- fmt.Fprintln(os.Stderr, "pprof endpoint never became reachable")
- os.Exit(1)
+// TestStartPProfBindFailureIsNonFatal pins the error path (adversarial): when
+// the port is taken, startPProf must return ErrPProfBindFailed and must NOT
+// panic or exit the process. A wrong implementation that swallows the error or
+// calls os.Exit would fail (process dies / wrong error type).
+func TestStartPProfBindFailureIsNonFatal(t *testing.T) {
+ addr := freeLocalAddr(t)
+ ln, err := net.Listen("tcp", addr)
+ if err != nil {
+ t.Fatalf("pre-occupy listen: %v", err)
}
- out := runPProfChild(t, "TestMainDeepDocProfileStartsPProf", envVar)
- if !strings.Contains(out, "Starting pprof server") {
- t.Fatalf("pprof startup line missing from child output:\n%s", out)
+ defer ln.Close()
+ if err := startPProf(addr); !errors.Is(err, ErrPProfBindFailed) {
+ t.Fatalf("startPProf on occupied %q = %v, want ErrPProfBindFailed", addr, err)
}
}
-// TestMainDeepDocWithoutProfileDoesNotStartPProf is the adversarial guard for
-// the flag gate: --deepdoc WITHOUT --profile must not open the profiling
-// port. An implementation that starts pprof unconditionally (ignoring
-// arguments.EnablePProf) passes the positive test above but fails here. The
-// child waits for main() to finish its early boot first, so a pprof server
-// started by that path would already be listening when the silence window
-// opens.
-func TestMainDeepDocWithoutProfileDoesNotStartPProf(t *testing.T) {
- const envVar = "RAGFLOW_TEST_DEEPDOC_PROFILE_NEGATIVE"
- if os.Getenv(envVar) == "1" {
- if pprofReachable() {
- os.Exit(pprofSkipExit)
- }
- os.Args = []string{os.Args[0], "--deepdoc"}
- select {
- case <-pprofChildBoot():
- case <-time.After(pprofBootWait):
- fmt.Fprintln(os.Stderr, "main() did not return in time")
- os.Exit(1)
- }
- deadline := time.Now().Add(pprofSettleWait)
- for time.Now().Before(deadline) {
- if pprofReachable() {
- fmt.Fprintln(os.Stderr, "pprof endpoint reachable without --profile")
- os.Exit(1)
- }
- time.Sleep(50 * time.Millisecond)
- }
- os.Exit(0)
+// TestPProfAddrDefault pins that pprofAddr() defaults to localhost:6060 when
+// RAGFLOW_PPROF_ADDR is unset.
+func TestPProfAddrDefault(t *testing.T) {
+ t.Setenv("RAGFLOW_PPROF_ADDR", "")
+ if got := pprofAddr(); got != "localhost:6060" {
+ t.Fatalf("pprofAddr() = %q, want localhost:6060", got)
}
- runPProfChild(t, "TestMainDeepDocWithoutProfileDoesNotStartPProf", envVar)
}
-// TestMainDeepDocProfilePProfBindFailureIsNonFatal pins the error path: when
-// the profiling port is already taken, the DeepDoc boot must log the failure
-// and keep running — the inference server matters more than profiling. The
-// child occupies localhost:6060 itself before booting main(), so main()'s
-// listener attempt is guaranteed to fail; the test passes only if main()
-// still returns without exiting the process.
-func TestMainDeepDocProfilePProfBindFailureIsNonFatal(t *testing.T) {
- const envVar = "RAGFLOW_TEST_DEEPDOC_PROFILE_BINDFAIL"
- if os.Getenv(envVar) == "1" {
- ln, err := net.Listen("tcp", "localhost:6060")
- if err != nil {
- os.Exit(pprofSkipExit)
- }
- defer ln.Close()
- os.Args = []string{os.Args[0], "--deepdoc", "--profile"}
- select {
- case <-pprofChildBoot():
- os.Exit(0)
- case <-time.After(pprofBootWait):
- fmt.Fprintln(os.Stderr, "main() did not return in time")
- os.Exit(1)
- }
+// TestPProfAddrFromEnv pins that pprofAddr() honors RAGFLOW_PPROF_ADDR
+// (adversarial: a distinguishing override). The old hard-coded localhost:6060
+// would make this fail, proving the configurability is real.
+func TestPProfAddrFromEnv(t *testing.T) {
+ const want = "127.0.0.1:0"
+ t.Setenv("RAGFLOW_PPROF_ADDR", want)
+ if got := pprofAddr(); got != want {
+ t.Fatalf("pprofAddr() = %q, want %q", got, want)
}
- runPProfChild(t, "TestMainDeepDocProfilePProfBindFailureIsNonFatal", envVar)
}