RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(go-ingestion): remove checkpoint/resume support (#19924)
xugangqiangcd2e548215b0
diff --git a/internal/ingestion/pipeline/cleanup.go b/internal/ingestion/pipeline/cleanup.goindex 4cb11705f3..40d5541ffa 100644--- a/internal/ingestion/pipeline/cleanup.go+++ b/internal/ingestion/pipeline/cleanup.go@@ -3,55 +3,17 @@ package pipelineimport ("context""errors"- "fmt"- "ragflow/internal/agent/canvas""ragflow/internal/ingestion/chunkcache")-// cleanupCheckpointState removes the checkpoint and the fingerprints that-// describe the graph which produced it. It returns storage failures to-// callers that need to gate a new run on cleanup completion; the pipeline's-// cancellation path logs those failures and remains best-effort.-func cleanupCheckpointState(ctx context.Context, store canvas.CheckPointStore, tracker *canvas.RunTracker, checkpointID string) error {- if checkpointID == "" {- return errors.New("checkpoint cleanup requires a checkpoint id")- }- var cleanupErr error- if store != nil {- for _, key := range []string{checkpointID, checkpointID + dslKeySuffix, checkpointID + ovfKeySuffix} {- if err := store.Delete(ctx, key); err != nil {- cleanupErr = errors.Join(cleanupErr, fmt.Errorf("delete %s: %w", key, err))- }- }- }- if tracker != nil {- if err := tracker.ClearInterruptID(ctx, checkpointID); err != nil {- cleanupErr = errors.Join(cleanupErr, fmt.Errorf("clear interrupt %s: %w", checkpointID, err))- }- }- return cleanupErr-}--// cleanupTaskState clears all resumable state associated with a task before-// its terminal task row is removed for a rerun. Cache entries are purged only-// after checkpoint cleanup succeeds so a failed cleanup remains retryable.-func cleanupTaskState(ctx context.Context, store canvas.CheckPointStore, tracker *canvas.RunTracker, taskID string) error {- if err := cleanupCheckpointState(ctx, store, tracker, taskID); err != nil {- return err- }- if err := chunkcache.PurgeTask(ctx, chunkcache.Client(), taskID); err != nil {- return fmt.Errorf("purge chunk cache: %w", err)- }- return nil-}--// PurgeTaskState removes checkpoint, tracker, and per-chunk cache state for a-// task using the process-wide Redis clients.+// PurgeTaskState removes the per-chunk cache state for a task using the+// process-wide Kvrocks client. The checkpoint (eino resume) store was removed+// from the ingestion pipeline, so only the per-chunk LLM/embedding results are+// purged here; a rerun therefore starts with a cold chunk cache.func PurgeTaskState(ctx context.Context, taskID string) error {if taskID == "" {return errors.New("task state cleanup requires a task id")}- p := &Pipeline{taskID: taskID}- return cleanupTaskState(ctx, p.resolveStore(), p.resolveTracker(), taskID)+ return chunkcache.PurgeTask(ctx, chunkcache.Client(), taskID)}diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.goindex d9aafc648a..43dce51ea3 100644--- a/internal/ingestion/pipeline/pipeline.go+++ b/internal/ingestion/pipeline/pipeline.go@@ -18,9 +18,6 @@ package pipelineimport ("context"- "crypto/sha256"- "encoding/hex"- "encoding/json""errors""fmt""time"@@ -30,73 +27,34 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common"kvrocks "ragflow/internal/engine/kvrocks"- "ragflow/internal/ingestion/component""ragflow/internal/ingestion/component/globals""ragflow/internal/utility"- "github.com/cloudwego/eino/compose""go.uber.org/zap")// Pipeline is a compiled ingestion canvas plus task-scoped metadata.type Pipeline struct {- taskID string- // rawDSL holds the canonical (key-sorted) JSON of the canvas DSL this- // pipeline was compiled from. It is the basis for the resume-time DSL- // fingerprint (guardDSLChange): any edit to the template DSL changes- // these bytes, so a stale checkpoint can be detected and discarded- // instead of being resumed against an incompatible graph.- rawDSL []byte+ taskID stringdocumentID string // owning document; numeric progress is mirrored back to// the document table for existing status projections.canvas *canvas.Canvas- store canvas.CheckPointStore // optional injected; nil -> resolve at Run- tracker *canvas.RunTracker // optional injected; nil -> resolve at Run- // requireResume, when true, makes Run refuse to start if no checkpoint- // store can be resolved (no injected store AND no global Kvrocks client).- // Plan §6.a M4: a deployment that cannot persist checkpoints must- // not silently degrade to a non-resumable run — it must surface a clear,- // distinguishable error so the caller knows resume is unavailable.- requireResume bool- factory runtime.ComponentFactory // optional instance-scoped component factory- sink ProgressSink // optional progress sink; nil -> drop events (DB-independent)+ tracker *canvas.RunTracker // optional injected; nil -> resolve at Run+ factory runtime.ComponentFactory // optional instance-scoped component factory+ sink ProgressSink // optional progress sink; nil -> drop events (DB-independent)}-// ErrResumeUnavailable is returned by Run when WithRequireResume is set but no-// checkpoint store can be resolved (plan §6.a M4). Callers can test for it with-// errors.Is to surface a "resume unavailable" condition instead of a generic-// failure (e.g. refuse to enqueue the task rather than start a non-resumable-// run).-var ErrResumeUnavailable = errors.New("resume unavailable: no checkpoint store (Redis down or not configured)")-// PipelineOption mutates a Pipeline before Run. Used to inject test doubles// (in-memory store / miniredis tracker) or dedicated Redis pools.type PipelineOption func(*Pipeline)-// WithCheckPointStore injects a checkpoint store. When unset, Run resolves-// one from the global Kvrocks client (and degrades to a non-resumable run when-// Redis is unavailable — plan §6.a).-func WithCheckPointStore(s canvas.CheckPointStore) PipelineOption {- return func(p *Pipeline) { p.store = s }-}--// WithRunTracker injects a RunTracker for interrupt-id persistence / crash-// recovery. When unset, Run resolves one from the global Kvrocks client.+// WithRunTracker injects a RunTracker for progress mirroring (terminal+// run-status marks). When unset, Run resolves one from the global Kvrocks+// client.func WithRunTracker(t *canvas.RunTracker) PipelineOption {return func(p *Pipeline) { p.tracker = t }}-// WithRequireResume makes Run refuse to start when no checkpoint store can be-// resolved (no injected store AND no global Kvrocks client). This is plan A: a-// deployment that cannot persist checkpoints must not silently-// degrade to a non-resumable run — it must surface a clear, distinguishable-// error (ErrResumeUnavailable) so the caller knows resume is unavailable.-// Production ingestion wiring sets this; unit tests leave it off to exercise-// the non-resumable runPlain fallback.-func WithRequireResume() PipelineOption {- return func(p *Pipeline) { p.requireResume = true }-}-// WithDocumentID binds the pipeline's owning document so numeric progress can// be mirrored back into the document table.// Pass the empty string to disable the mirror (e.g. headless/test runs where@@ -153,17 +111,9 @@ func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pipif err := ValidatePipeline(cnv); err != nil {return nil, err}- // Capture the canonical canvas DSL bytes for the resume-time DSL- // fingerprint. json.Marshal sorts map keys, so this is stable across- // re-decodes of the same logical DSL (formatting-independent).- rawBytes, err := json.Marshal(canvasDSL)- if err != nil {- return nil, fmt.Errorf("pipeline: canonicalize DSL: %w", err)- }p := &Pipeline{taskID: taskID,canvas: cnv,- rawDSL: rawBytes,}for _, o := range opts {o(p)@@ -224,172 +174,10 @@ func cloneMapOrEmpty(m map[string]any) map[string]any {return out}-// defaultCheckpointTTL is the expiry applied to the eino checkpoint payload-// and the RunTracker hash. A finished run's checkpoint is deleted on success;-// the TTL only guards against leaks from crashed runs that never clean up.-//-// It matches the per-chunk cache TTL (chunkcache.TTL) so a crashed run leaves-// both its checkpoint and the chunk cache entries to expire on the same horizon.-var defaultCheckpointTTL = 7 * 24 * time.Hour--// dslKeySuffix / ovfKeySuffix derive the two DSL-fingerprint keys from a-// checkpoint id. Both live in the same CheckPointStore as the eino payload-// (under cpID+dslKeySuffix / cpID+ovfKeySuffix) so a resume can compare the-// current DSL against the one that wrote the checkpoint — without depending-// on the RunTracker being injected. eino only ever reads/writes the exact-// cpID, so these sibling keys never collide with eino's checkpoint.-//-// The split is deliberate: dslKeySuffix fingerprints the DSL FILE (the full-// canvas DSL — topology, component params, everything the user edits in the-// template), while ovfKeySuffix fingerprints only the runtime override_params-// (Doc.ParserConfig + injected LLM id). This lets the mismatch warning say-// whether the DSL file changed or the runtime override changed, instead of an-// ambiguous "params changed".-const (- dslKeySuffix = ":dsl"- ovfKeySuffix = ":ovf"-)--// dslFileFingerprint returns a stable hash of the FULL canvas DSL this-// pipeline was compiled from. It captures topology (components, edges, graph-// grouping) AND the per-component params/defaults — i.e. everything the user-// can edit in the DSL template file. The raw DSL bytes are key-sorted at-// decode time, so logical edits (not whitespace) drive the value.-//-// It returns an error (instead of a silent fallback) when rawDSL is unset:-// NewPipelineFromDSL always populates rawDSL, so an empty value means the-// Pipeline was built through an unexpected path. A fallback that re-encodes-// p.canvas would produce a different hash (Canvas.NodeParents is json:"-",-// and struct field order differs from the map form), silently breaking the-// change-detection contract.-func (p *Pipeline) dslFileFingerprint() (string, error) {- if len(p.rawDSL) == 0 {- return "", fmt.Errorf("rawDSL not set (Pipeline built without NewPipelineFromDSL?)")- }- h := sha256.New()- h.Write(p.rawDSL)- return hex.EncodeToString(h.Sum(nil)), nil-}--// overrideFingerprint returns a stable hash of the run-level override_params-// (Doc.ParserConfig + injected LLM id). It is tracked separately from the DSL-// file so a change limited to the runtime override is distinguishable (in the-// warning log) from an edit to the DSL template file.-func overrideFingerprint(override map[string]any) string {- h := sha256.New()- _ = json.NewEncoder(h).Encode(override)- return hex.EncodeToString(h.Sum(nil))-}--// classifyDSLChange decides why a stored fingerprint pair no longer matches-// the current run, for the mismatch warning log. It returns "" when neither-// changed (no stale checkpoint to discard). A DSL file change takes precedence-// in the label because an edited template is the more common/expected case;-// when both changed the label still says DSL. The caller only invokes this-// when a checkpoint actually exists, so an empty stored fingerprint is treated-// as "changed" (safe: discard and re-run from scratch).-func classifyDSLChange(storedDsl, storedOvf, dslFP, ovfFP string) string {- dslChanged := storedDsl != dslFP- ovfChanged := storedOvf != ovfFP- if !dslChanged && !ovfChanged {- return ""- }- if dslChanged {- return "DSL/template changed"- }- return "runtime override/parser-config changed"-}--// guardDSLChange prevents resuming a checkpoint written by a DIFFERENT DSL-// than the one currently compiled. eino checkpoints are bound to the graph's-// node ids / wiring; resuming against a modified graph restores state for-// nodes that no longer exist (or have different wiring) and makes eino error-// out. When either the DSL file fingerprint or the runtime override-// fingerprint differs we drop the stale checkpoint (and the persisted-// interrupt marker) so the run starts fresh from the entry node. When no-// checkpoint exists this is simply a fresh run, and we record both-// fingerprints so a later resume can detect edits. All store writes are-// best-effort: a write failure must not abort the run.-func (p *Pipeline) guardDSLChange(ctx context.Context, store canvas.CheckPointStore, tracker *canvas.RunTracker, cpID string, override map[string]any) {- // Only meaningful when a checkpoint store is wired (the resumable path).- // The non-resumable runPlain path never calls here, but guard anyway so a- // future caller cannot nil-panic on store.Get.- if store == nil {- return- }- dslKey := cpID + dslKeySuffix- ovfKey := cpID + ovfKeySuffix-- dslFP, err := p.dslFileFingerprint()- if err != nil {- // Without a DSL fingerprint we cannot detect edits. Skip the guard- // rather than risk an empty hash that silently mismatches every run.- // The run still proceeds (best-effort); resume safety is unchanged- // only because rawDSL is always set via NewPipelineFromDSL.- common.Error(fmt.Sprintf("pipeline: skip DSL-change guard for %s: %v", p.taskID, err), err)- return- }- ovfFP := overrideFingerprint(override)-- // A checkpoint from a previous run exists → this is a resume candidate.- // A read error means we cannot trust the existence check, so bail out- // entirely: do NOT overwrite cpID:dsl / cpID:ovf. Writing current- // fingerprints after a failed lookup could mask a real stale-checkpoint- // mismatch on a later resume (the old checkpoint would survive, but its- // fingerprint would be overwritten to match the new DSL).- _, found, cpErr := store.Get(ctx, cpID)- if cpErr != nil {- common.Error(fmt.Sprintf("pipeline: lookup checkpoint for %s failed, skip DSL-change guard: %v", p.taskID, cpErr), cpErr)- return- }- if found {- storedDsl, dOk, dErr := store.Get(ctx, dslKey)- storedOvf, oOk, oErr := store.Get(ctx, ovfKey)- if dErr != nil || oErr != nil {- common.Error(fmt.Sprintf("pipeline: read DSL fingerprints for %s failed, skip DSL-change guard: %v", p.taskID, coalesceErr(dErr, oErr)), coalesceErr(dErr, oErr))- return- }- reason := classifyDSLChange(orEmpty(dOk, storedDsl), orEmpty(oOk, storedOvf), dslFP, ovfFP)- if reason != "" {- common.Warn(fmt.Sprintf("pipeline: DSL for task %s changed since checkpoint was written (%s); discarding stale checkpoint and re-running from scratch", p.taskID, reason))- if err := store.Delete(ctx, cpID); err != nil {- common.Error(fmt.Sprintf("pipeline: delete stale checkpoint for %s failed: %v", p.taskID, err), err)- }- if tracker != nil {- if err := tracker.ClearInterruptID(ctx, cpID); err != nil {- common.Error(fmt.Sprintf("pipeline: clear interrupt id for %s failed: %v", p.taskID, err), err)- }- }- }- }- // Record the fingerprints of the DSL + override that produced this run.- if err := store.Set(ctx, dslKey, []byte(dslFP)); err != nil {- common.Error(fmt.Sprintf("pipeline: persist DSL fingerprint for %s failed: %v", p.taskID, err), err)- }- if err := store.Set(ctx, ovfKey, []byte(ovfFP)); err != nil {- common.Error(fmt.Sprintf("pipeline: persist override fingerprint for %s failed: %v", p.taskID, err), err)- }-}--// orEmpty returns the byte slice when ok, else an empty string, so a missing-// stored fingerprint is treated as "" (classifyDSLChange then sees a change).-func orEmpty(ok bool, b []byte) string {- if !ok {- return ""- }- return string(b)-}--// coalesceErr returns the first non-nil error, so a single message can report-// whichever of two reads failed without dereferencing nil.-func coalesceErr(errs ...error) error {- for _, e := range errs {- if e != nil {- return e- }- }- return nil-}+// defaultTrackerTTL is the expiry applied to the RunTracker run-status hash.+// It matches the per-chunk cache TTL (chunkcache.TTL) so both expire on the+// same horizon.+var defaultTrackerTTL = 7 * 24 * time.Hour// Run executes the full ingestion graph described by the canonical DSL.// There is no pipeline-layer partial resume entry point: execution always@@ -414,43 +202,15 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParamcompileCtx = canvas.WithComponentFactory(compileCtx, p.factory)}- // Resolve the checkpoint store + run tracker. Resume (interrupt-after- // non-terminal node) requires a checkpoint store to persist the resume- // point; without one we fall back to a single non-resumable Invoke- // (plan §6.a degrade — progress stays observable, the run just cannot- // pause/resume across nodes).- store := p.resolveStore()+ // Resolve the run tracker for progress mirroring (terminal+ // MarkSucceeded/MarkFailed/MarkCancelled marks). Resume-from-checkpoint is+ // intentionally not supported: the pipeline always runs the full graph+ // from the entry node, and component-level replay decisions (e.g. serving+ // prior LLM/embedding results from the per-chunk cache) belong to the+ // components themselves.tracker := p.resolveTracker()- // M4 (plan §6.a): refuse to start when resume is required but no- // checkpoint store is resolvable. A Redis-less deployment must not pretend- // the task is resumable; it must report the gap clearly so the caller can- // refuse to enqueue the task instead of silently running a non-resumable- // run (which would violate "re-run same task, completed components not- // redone").- if p.requireResume && store == nil {- return nil, fmt.Errorf("pipeline: Run: %w", ErrResumeUnavailable)- }- resumable := store != nil-var compileOpts []canvas.CompileOption- if resumable {- compileOpts = append(compileOpts,- canvas.WithCheckPointStore(store),- canvas.WithCheckPointID(p.taskID),- )- // Single interrupt point: immediately after the Parser component. The- // Extractor / Tokenizer stages do NOT take a checkpoint of their own —- // a resume re-runs them, but the per-chunk cache (chunkcache) absorbs- // the cost by serving prior LLM/embedding results. Interruption after- // every non-terminal component was dropped on purpose to avoid one- // needless ResumeWithData round per stage.- if parserCpnID := ExtractParserCpnID(p.rawDSL, component.ComponentNameParser); parserCpnID != "" {- compileOpts = append(compileOpts, canvas.WithInterruptAfter([]string{parserCpnID}))- } else {- compileOpts = append(compileOpts, canvas.WithInterruptAfterNonTerminalCpn())- }- }// Run-level setups (keyed by cpnID) override the DSL-baked component// setups at compile time (higher priority; see canvas.WithOverrideParams).if overrideParams != nil {@@ -495,50 +255,24 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam// no-op, leaving orphaned cache entries until TTL expiry.globals.SetTaskID(runCtx, p.taskID)- if !resumable {- return p.runPlain(runCtx, current, compiled, tracker, runState)- }-- // Resumable path: detect DSL / override edits since the checkpoint was- // written and discard a stale checkpoint before resuming (see guardDSLChange).- p.guardDSLChange(ctx, store, tracker, p.taskID, overrideParams)-- // Resumable path: record the run, then loop Invoke until the graph- // completes or a non-resumable error surfaces.- if tracker != nil {- if err = tracker.Start(ctx, p.taskID, "", "", ""); err != nil {- common.Error(fmt.Sprintf("pipeline: RunTracker.Start for task %s failed: %v", p.taskID, err), err)- }- }- return p.runResumable(ctx, runCtx, current, compiled, store, tracker, runState)-}--// resolveStore returns the injected store, or a Kvrocks-backed one when the-// global Kvrocks client is available. Returns nil (degraded, non-resumable)-// when neither is present.-func (p *Pipeline) resolveStore() canvas.CheckPointStore {- if p.store != nil {- return p.store- }- if kvrocks.Get() != nil {- return canvas.NewKvrocksCheckPointStore(defaultCheckpointTTL)- }- return nil+ return p.runPlain(runCtx, current, compiled, tracker, runState)}-// resolveTracker mirrors resolveStore for the RunTracker.+// resolveTracker resolves the RunTracker used for progress mirroring. When an+// instance-scoped tracker was injected it wins; otherwise one is created on+// the global Kvrocks client when available.func (p *Pipeline) resolveTracker() *canvas.RunTracker {if p.tracker != nil {return p.tracker}if kvrocks.Get() != nil {- return canvas.NewRunTracker(defaultCheckpointTTL)+ return canvas.NewRunTracker(defaultTrackerTTL)}return nil}-// runPlain executes a single Invoke with no checkpoint/resume. Used when no-// checkpoint store is available; progress is still recorded via the sink.+// runPlain executes the full ingestion graph with a single Invoke and records+// the terminal run status via the sink/RunTracker when available.func (p *Pipeline) runPlain(runCtx context.Context, current map[string]any, compiled *canvas.CompiledCanvas, tracker *canvas.RunTracker, runState *canvas.CanvasState) (map[string]any, error) {// Terminal tracker writes must survive a run ctx that gets cancelled// (the common failure/cancel path). Derive a fresh detached ctx per@@ -573,113 +307,8 @@ func (p *Pipeline) runPlain(runCtx context.Context, current map[string]any, compreturn finalizeResult(current, out, runState), nil}-// runResumable drives the graph with eino's interrupt-after-node + resume-// loop (plan §8 step 3). Every non-terminal-node pause is auto-resumed with-// nil data (ingestion resume needs no user input). The loop's TOP reads any-// persisted interrupt id — from the RunTracker (cross-process crash-// recovery) or an in-process fallback — and resumes; the BOTTOM only persists-// the id, never inline-resumes (avoids double-resuming one ctx, plan §4.2-// 建议2).-func (p *Pipeline) runResumable(ctx context.Context, runCtx context.Context, current map[string]any, compiled *canvas.CompiledCanvas, store canvas.CheckPointStore, tracker *canvas.RunTracker, runState *canvas.CanvasState) (map[string]any, error) {- cpID := compiled.CheckPointID- var localInterruptID string // in-process resume fallback when tracker is nil-- const maxResumeRounds = 1000- for round := 0; round < maxResumeRounds; round++ {- // TOP: recover the pending interrupt (crash recovery or in-process).- resumeID := ""- if tracker != nil {- if id, ok, _ := tracker.GetInterruptID(ctx, cpID); ok && id != "" {- resumeID = id- }- }- if resumeID == "" {- resumeID = localInterruptID- }- if resumeID != "" {- // Mark the resume target. Keep passing the full original input- // instead of nil: when eino loads the checkpoint it ignores the- // passed input entirely (restoreCheckPointState only restores- // channels/state, restoreTasks replays cp.Inputs), and when the- // checkpoint is missing it silently starts a FRESH run from the- // entry node — so nil input would re-execute the source node- // (File) without doc_id / file[0].name and fail with "inputs- // missing". Passing `current` covers both cases with no- // check-then-use race.- runCtx = compose.ResumeWithData(runCtx, resumeID, nil)- }-- out, invokeErr := compiled.Workflow.Invoke(runCtx, current, compose.WithCheckPointID(cpID))- if invokeErr == nil {- // Terminal-state writes must survive a ctx cancelled around the- // Invoke boundary. Derive a fresh detached ctx here — bounded by- // a short timeout so a hung Redis cannot stall completion — so- // the 5s window covers only the cleanup, not the whole run- // (mirrors markStopped/markFailed in ingestion_service.go).- stateCtx, stateCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)- defer stateCancel()- if tracker != nil {- utility.BestEffort(fmt.Sprintf("ClearInterruptID for %s", p.taskID), func() error { return tracker.ClearInterruptID(stateCtx, cpID) })- utility.BestEffort(fmt.Sprintf("MarkSucceeded for %s", p.taskID), func() error { return tracker.MarkSucceeded(stateCtx, cpID) })- }- if store != nil {- utility.BestEffort(fmt.Sprintf("delete checkpoint for %s", p.taskID), func() error { return store.Delete(stateCtx, cpID) })- utility.BestEffort(fmt.Sprintf("delete DSL fingerprint for %s", p.taskID), func() error { return store.Delete(stateCtx, cpID+dslKeySuffix) })- utility.BestEffort(fmt.Sprintf("delete override fingerprint for %s", p.taskID), func() error { return store.Delete(stateCtx, cpID+ovfKeySuffix) })- }- return finalizeResult(current, out, runState), nil- }-- // Cancellation (plan §4.3.b): wipe the checkpoint and mark cancelled.- if errors.Is(ctx.Err(), context.Canceled) || errors.Is(ctx.Err(), context.DeadlineExceeded) {- // The run ctx is already cancelled, so Redis writes with it would- // fail immediately. Derive a fresh detached ctx so cleanup +- // MarkCancelled actually land; the 5s window covers the cleanup- // only, regardless of how long the run itself took.- cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)- defer cleanupCancel()- p.cleanupCheckpoint(cleanupCtx, store, tracker, cpID)- if tracker != nil {- utility.BestEffort(fmt.Sprintf("MarkCancelled for %s", p.taskID), func() error { return tracker.MarkCancelled(cleanupCtx, cpID) })- }- return current, fmt.Errorf("pipeline: run cancelled: %w", ctx.Err())- }-- if !canvas.IsInterruptError(invokeErr) {- // Same detached-ctx guarantee as the success/cancel paths: the- // run ctx may be cancelled by the time the failure surfaces.- failCtx, failCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)- defer failCancel()- if tracker != nil {- utility.BestEffort(fmt.Sprintf("MarkFailed for %s", p.taskID), func() error { return tracker.MarkFailed(failCtx, cpID, invokeErr.Error()) })- }- return current, fmt.Errorf("pipeline: run canvas workflow: %w", invokeErr)- }-- // Paused at a non-terminal node: persist for crash recovery, then- // resume on the next loop iteration's TOP.- ctxs := canvas.ExtractInterruptContexts(invokeErr)- id := canvas.FirstInterruptID(ctxs)- localInterruptID = id- if tracker != nil {- if err := tracker.AttachInterrupt(ctx, cpID, id); err != nil {- common.Error(fmt.Sprintf("pipeline: AttachInterrupt for task %s failed: %v", p.taskID, err), err)- }- }- }- return current, fmt.Errorf("pipeline: run exceeded max resume rounds (%d) for task %s", maxResumeRounds, p.taskID)-}--// cleanupCheckpoint wipes the eino checkpoint payload and the persisted-// interrupt id (plan §4.3.b cancelled path).-func (p *Pipeline) cleanupCheckpoint(ctx context.Context, store canvas.CheckPointStore, tracker *canvas.RunTracker, cpID string) {- if err := cleanupCheckpointState(ctx, store, tracker, cpID); err != nil {- common.Error(fmt.Sprintf("pipeline: cleanup checkpoint %s failed: %v", cpID, err), err)- }-}-// finalizeResult merges the graph output into the input map and attaches the-// canvas state snapshot — the shared success payload for both run paths.+// canvas state snapshot as the success payload.func finalizeResult(current, out map[string]any, runState *canvas.CanvasState) map[string]any {if out == nil {current["state"] = runState.Snapshot()diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.goindex 8af84ed3ec..57b7c50f42 100644--- a/internal/ingestion/pipeline/pipeline_test.go+++ b/internal/ingestion/pipeline/pipeline_test.go@@ -18,7 +18,6 @@ package pipelineimport ("context"- "errors""fmt""strings""sync"@@ -51,25 +50,6 @@ func (m *mockCanvasStage) Invoke(_ context.Context, _ *gorm.DB, inputs map[strinfunc (m *mockCanvasStage) Inputs() map[string]string { return map[string]string{"name": "string"} }func (m *mockCanvasStage) Outputs() map[string]string { return map[string]string{"output": "any"} }-// oneShotErrStage errors on the first Invoke (simulating a component crash-// mid-run), then delegates to the embedded mock on subsequent calls. Used to-// test that a second pipeline Run on the same taskID resumes past non-terminal-// checkpoints instead of re-executing completed components.-type oneShotErrStage struct {- mockCanvasStage- n int-}--func (s *oneShotErrStage) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {- s.n++- if s.n == 1 {- return nil, errors.New("simulated crash")- }- return s.mockCanvasStage.Invoke(ctx, db, inputs)-}-func (s *oneShotErrStage) Inputs() map[string]string { return s.mockCanvasStage.Inputs() }-func (s *oneShotErrStage) Outputs() map[string]string { return s.mockCanvasStage.Outputs() }-func TestPipelineRunHappyPath(t *testing.T) {stageA := &mockCanvasStage{output: map[string]any{"a": 1}}stageB := &mockCanvasStage{output: map[string]any{"b": 2}}@@ -189,48 +169,6 @@ func (s *factorySentinelStage) Invoke(_ context.Context, _ *gorm.DB, inputs map[return out, nil}-// memCheckpointStore is a thread-safe in-memory canvas.CheckPointStore used-// to exercise the resumable run path without Redis.-type memCheckpointStore struct {- mu sync.Mutex- data map[string][]byte- deleted int // number of times Delete was called-}--func newMemCheckpointStore() *memCheckpointStore {- return &memCheckpointStore{data: map[string][]byte{}}-}--func (s *memCheckpointStore) Get(_ context.Context, id string) ([]byte, bool, error) {- s.mu.Lock()- defer s.mu.Unlock()- v, ok := s.data[id]- return v, ok, nil-}--func (s *memCheckpointStore) Set(_ context.Context, id string, payload []byte) error {- s.mu.Lock()- defer s.mu.Unlock()- cp := make([]byte, len(payload))- copy(cp, payload)- s.data[id] = cp- return nil-}--func (s *memCheckpointStore) Delete(_ context.Context, id string) error {- s.mu.Lock()- defer s.mu.Unlock()- delete(s.data, id)- s.deleted++- return nil-}--func (s *memCheckpointStore) deleteCount() int {- s.mu.Lock()- defer s.mu.Unlock()- return s.deleted-}-// TestPipelineRun_InstanceFactoryOverridesDefaultFactory verifies that a// pipeline-scoped component factory can provide task-specific components.func TestPipelineRun_InstanceFactoryOverridesDefaultFactory(t *testing.T) {@@ -336,604 +274,6 @@ func TestPipelineRun_TaskScopedFactoriesDoNotLeakAcrossConcurrentPipelines(t *te}}-func TestPipelineRunResumableAutoResumes(t *testing.T) {- stageA := &mockCanvasStage{output: map[string]any{"a": 1}}- stageB := &mockCanvasStage{output: map[string]any{"b": 2}}-- const (- nameA = "p.ResumeStageA"- nameB = "p.ResumeStageB"- )- runtime.MustRegister(nameA, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return stageA, nil },- runtime.Metadata{Version: "1.0.0"})- runtime.MustRegister(nameB, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return stageB, nil },- runtime.Metadata{Version: "1.0.0"})-- pipe, err := NewPipelineFromDSL([]byte(`{- "dsl": {- "components": {- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}- },- "path": ["begin", "a", "b"],- "graph": {"nodes": []}- }- }`), "task-resume", WithCheckPointStore(newMemCheckpointStore()))- if err != nil {- t.Fatalf("NewPipelineFromDSL: %v", err)- }-- out, err := pipe.Run(t.Context(), map[string]any{"name": "doc-resume"}, nil)- if err != nil {- t.Fatalf("Run: %v", err)- }- if !stageA.called || !stageB.called {- t.Fatalf("expected both stages to run, got A=%v B=%v", stageA.called, stageB.called)- }- // No re-run on resume: each node must execute exactly once.- if stageA.calls != 1 || stageB.calls != 1 {- t.Fatalf("expected each stage to run exactly once, got A.calls=%d B.calls=%d", stageA.calls, stageB.calls)- }- if out == nil {- t.Fatal("expected non-nil output")- }-}--// TestPipelineRunResumableCrossRunResume validates crash-recovery resume-// across two Run calls: when the terminal component errors mid-run (simulated-// crash), non-terminal checkpoints + interrupt state persist. The second Run-// on the same taskID resumes past completed non-terminal components instead of-// re-executing them. A non-terminal stage (A) runs exactly once across both runs; the-// terminal stage (B, a oneShotErrStage) errors on run 1 and succeeds on run 2.-func TestPipelineRunResumableCrossRunResume(t *testing.T) {- mockA := &mockCanvasStage{output: map[string]any{"a": 1}}- mockB := &mockCanvasStage{output: map[string]any{"b": 2}}- termStage := &oneShotErrStage{mockCanvasStage: *mockB}-- const (- nameA = "p.XRunStageA"- nameB = "p.XRunStageB"- )- runtime.MustRegister(nameA, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return mockA, nil },- runtime.Metadata{Version: "1.0.0"})- runtime.MustRegister(nameB, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },- runtime.Metadata{Version: "1.0.0"})-- store := newMemCheckpointStore()- mr := miniredis.RunT(t)- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})- t.Cleanup(func() { client.Close() })- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)-- const taskID = "task-cross-run-resume"- pipe, err := NewPipelineFromDSL([]byte(`{- "dsl": {- "components": {- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}- },- "path": ["begin", "a", "b"],- "graph": {"nodes": []}- }- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))- if err != nil {- t.Fatalf("NewPipelineFromDSL: %v", err)- }-- // Run 1: terminal B errors (oneShotErrStage n=1), simulating a crash.- // Non-terminal A's checkpoint + interrupt persist because the error path- // does not call ClearInterruptID or store.Delete.- _, err = pipe.Run(t.Context(), map[string]any{"name": "doc-cross-run"}, nil)- if err == nil {- t.Fatal("Run 1: expected error from simulated crash, got nil")- }- if mockA.calls != 1 {- t.Fatalf("Run 1: expected A to run once, got %d", mockA.calls)- }- // oneShotErrStage did not delegate to its embedded mock on the first call.- if termStage.calls != 0 {- t.Fatalf("Run 1: expected B (embedded mock) calls=0 (error before delegate), got %d", termStage.calls)- }-- // Run 2: resume from after A via tracker.GetInterruptID. A is skipped;- // B's oneShotErrStage (n=2) delegates to its embedded mock successfully.- _, err = pipe.Run(t.Context(), map[string]any{"name": "doc-cross-run"}, nil)- if err != nil {- t.Fatalf("Run 2: expected recovery success, got error: %v", err)- }- if mockA.calls != 1 {- t.Fatalf("Run 2: expected A to still have calls=1 (was skipped by resume), got %d", mockA.calls)- }- if termStage.calls != 1 {- t.Fatalf("Run 2: expected B (embedded mock) calls=1 (delegated once), got %d", termStage.calls)- }-}--// docIDGuardStage mirrors the File component's contract: it requires-// doc_id (or an explicit file descriptor) in its input and fails otherwise.-// It lets the resume tests observe whether the graph was re-entered with the-// original run input or with a nil input (the "inputs missing" failure).-type docIDGuardStage struct {- mockCanvasStage- mu sync.Mutex- guardCalls int- missingInputs int-}--func (m *docIDGuardStage) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {- m.mu.Lock()- m.guardCalls++- m.mu.Unlock()- if inputs == nil || inputs["doc_id"] == nil {- m.mu.Lock()- m.missingInputs++- m.mu.Unlock()- return nil, errors.New("file: inputs missing doc_id or file[0].name")- }- return m.mockCanvasStage.Invoke(ctx, db, inputs)-}--func (m *docIDGuardStage) snapshot() (calls, missing int) {- m.mu.Lock()- defer m.mu.Unlock()- return m.guardCalls, m.missingInputs-}--// TestPipelineRunResumableOrphanInterruptIDWithoutCheckpoint covers the-// cancel-race state: the interrupt id survives in the RunTracker but the-// checkpoint payload is gone. Before the fix, runResumable treated the-// orphaned id as "resume" and invoked the graph with nil input, which-// re-entered the source node (docIDGuardStage) and failed with-// "inputs missing doc_id". The fix keeps passing the full original input on-// resume rounds, so the graph starts fresh with doc_id intact instead of-// re-entering the source node with nil input.-func TestPipelineRunResumableOrphanInterruptIDWithoutCheckpoint(t *testing.T) {- guard := &docIDGuardStage{mockCanvasStage: mockCanvasStage{output: map[string]any{"a": 1}}}- termStage := &oneShotErrStage{mockCanvasStage: mockCanvasStage{output: map[string]any{"b": 2}}}-- const (- nameA = "p.OrphanA"- nameB = "p.OrphanB"- )- runtime.MustRegister(nameA, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return guard, nil },- runtime.Metadata{Version: "1.0.0"})- runtime.MustRegister(nameB, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },- runtime.Metadata{Version: "1.0.0"})-- const taskID = "task-orphan-interrupt"-- store := newMemCheckpointStore()- mr := miniredis.RunT(t)- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})- t.Cleanup(func() {- if err := client.Close(); err != nil {- t.Errorf("client.Close: %v", err)- }- })- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)-- newPipe := func() *Pipeline {- pipe, err := NewPipelineFromDSL([]byte(`{- "dsl": {- "components": {- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}- },- "path": ["begin", "a", "b"],- "graph": {"nodes": []}- }- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))- if err != nil {- t.Fatalf("NewPipelineFromDSL: %v", err)- }- return pipe- }-- // Run 1: terminal B errors, leaving a checkpoint + interrupt id.- pipe1 := newPipe()- _, err := pipe1.Run(context.Background(), map[string]any{"doc_id": "d1"}, nil)- if err == nil {- t.Fatal("Run 1: expected error from simulated crash, got nil")- }- if calls, missing := guard.snapshot(); calls != 1 || missing != 0 {- t.Fatalf("Run 1: guard.calls=%d missing=%d, want 1/0", calls, missing)- }-- // Simulate the cancel-race: the checkpoint payload vanishes but the- // interrupt id stays behind (cleanup used a cancelled ctx and failed).- if err := store.Delete(context.Background(), taskID); err != nil {- t.Fatalf("store.Delete: %v", err)- }- if _, ok, _ := tracker.GetInterruptID(context.Background(), taskID); !ok {- t.Fatal("precondition: interrupt id must still be present after checkpoint deletion")- }-- // Run 2: must start fresh with the full input preserved (no nil-input- // re-entry). The guard stage must see doc_id on every invocation- // (no "inputs missing").- pipe2 := newPipe()- _, err = pipe2.Run(context.Background(), map[string]any{"doc_id": "d1"}, nil)- if err != nil {- t.Fatalf("Run 2: expected success after checkpoint loss, got: %v", err)- }- calls, missing := guard.snapshot()- if missing != 0 {- t.Fatalf("Run 2: source stage ran %d times with missing doc_id, want 0 (orphan interrupt id must not nil the input)", missing)- }- if calls != 2 {- t.Fatalf("Run 2: source stage calls=%d, want 2 (fresh run after checkpoint loss)", calls)- }-}--// longRunCancelStage simulates a long-running component (e.g. a PDF parser):-// it blocks until the run ctx is cancelled, then returns a cancellation error-// so runResumable takes the cancel branch. The started channel lets the test-// coordinate the cancel so it lands AFTER the entry-derived stateCtx's 5s-// window would have expired — reproducing the "run took longer than the-// detached-cleanup budget" failure mode.-type longRunCancelStage struct {- started chan struct{}-}--func (m *longRunCancelStage) Invoke(ctx context.Context, _ *gorm.DB, inputs map[string]any) (map[string]any, error) {- if m.started != nil {- close(m.started)- }- <-ctx.Done()- return nil, ctx.Err()-}-func (m *longRunCancelStage) Inputs() map[string]string { return map[string]string{"name": "string"} }-func (m *longRunCancelStage) Outputs() map[string]string { return map[string]string{"output": "any"} }--// TestPipelineRunResumableCancelCleanupAfterLongRun covers the detached-ctx-// timing defect: the previous "unified" stateCtx was derived once at the top-// of runResumable with a 5s timeout, so any run lasting longer than 5s-// reached the cancel branch with an already-expired ctx and the cleanup-// (checkpoint delete + ClearInterruptID + MarkCancelled) silently failed —-// reintroducing the exact leak this PR fixes. Each terminal branch must-// derive its own detached ctx at the point of cleanup.-func TestPipelineRunResumableCancelCleanupAfterLongRun(t *testing.T) {- started := make(chan struct{})- longStage := &longRunCancelStage{started: started}- termStage := &mockCanvasStage{output: map[string]any{"b": 2}}-- const (- nameA = "p.LongCancelA"- nameB = "p.LongCancelB"- )- runtime.MustRegister(nameA, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return longStage, nil },- runtime.Metadata{Version: "1.0.0"})- runtime.MustRegister(nameB, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },- runtime.Metadata{Version: "1.0.0"})-- const taskID = "task-long-run-cancel"-- store := newMemCheckpointStore()- mr := miniredis.RunT(t)- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})- t.Cleanup(func() {- if err := client.Close(); err != nil {- t.Errorf("client.Close: %v", err)- }- })- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)-- pipe, err := NewPipelineFromDSL([]byte(`{- "dsl": {- "components": {- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}- },- "path": ["begin", "a", "b"],- "graph": {"nodes": []}- }- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))- if err != nil {- t.Fatalf("NewPipelineFromDSL: %v", err)- }-- runCtx, cancel := context.WithCancel(context.Background())- go func() {- <-started- // Simulate a long parse: cancel only after the entry-derived 5s- // cleanup window would have expired. 6s > 5s. Keep the test fast by- // timing the sleep from component start rather than from Run entry.- time.Sleep(6 * time.Second)- cancel()- }()-- _, err = pipe.Run(runCtx, map[string]any{"doc_id": "d1"}, nil)- if err == nil {- t.Fatal("expected cancellation error, got nil")- }-- // The cancel branch must have cleaned up with a fresh detached ctx.- if _, ok, _ := store.Get(context.Background(), taskID); ok {- t.Error("checkpoint was not deleted after cancellation (detached-ctx cleanup failed)")- }- if _, ok, _ := tracker.GetInterruptID(context.Background(), taskID); ok {- t.Error("interrupt id was not cleared after cancellation (detached-ctx cleanup failed)")- }- fields, err := tracker.Get(context.Background(), taskID)- if err != nil {- t.Fatalf("tracker.Get: %v", err)- }- if status := fields["status"]; status != "3" {- t.Errorf("tracker status = %q, want %q (cancelled)", status, "3")- }-}--// TestPipelineRunResumableDSLChanged discards a stale checkpoint when the DSL-// is edited between the failed run and the resume. Run 1 fails on the terminal-// stage, persisting a checkpoint keyed by taskID. Run 2 re-runs the SAME-// taskID but with a modified DSL (a param edit, enough to change the-// fingerprint). The guard must detect the mismatch, discard the checkpoint,-// and start fresh — so every stage re-executes (A runs on both runs) instead-// of erroring while resuming an incompatible graph (the original bug).-func TestPipelineRunResumableDSLChanged(t *testing.T) {- mockA := &mockCanvasStage{output: map[string]any{"a": 1}}- termStage := &oneShotErrStage{mockCanvasStage: mockCanvasStage{output: map[string]any{"b": 2}}}-- const (- nameA = "p.DSLChangedA"- nameB = "p.DSLChangedB"- )- runtime.MustRegister(nameA, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return mockA, nil },- runtime.Metadata{Version: "1.0.0"})- runtime.MustRegister(nameB, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },- runtime.Metadata{Version: "1.0.0"})-- const taskID = "task-dsl-changed"-- // Shared store + tracker across both runs (same taskID).- store := newMemCheckpointStore()- mr := miniredis.RunT(t)- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})- t.Cleanup(func() { client.Close() })- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)-- // Run 1: original DSL; terminal B errors (oneShotErrStage n=1), leaving a- // checkpoint + interrupt for the original DSL fingerprint.- pipe1, err := NewPipelineFromDSL([]byte(`{- "dsl": {- "components": {- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}- },- "path": ["begin", "a", "b"],- "graph": {"nodes": []}- }- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))- if err != nil {- t.Fatalf("NewPipelineFromDSL (run 1): %v", err)- }- if _, err := pipe1.Run(t.Context(), map[string]any{"name": "doc-dsl-changed"}, nil); err == nil {- t.Fatal("Run 1: expected error from simulated crash, got nil")- }- if mockA.calls != 1 {- t.Fatalf("Run 1: expected A to run once, got %d", mockA.calls)- }-- // Run 2: SAME taskID, but a modified DSL (param added to "a" → different- // fingerprint). Without the guard this would resume the stale checkpoint- // against an incompatible graph and error; with the guard it discards and- // re-runs from scratch.- pipe2, err := NewPipelineFromDSL([]byte(`{- "dsl": {- "components": {- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},- "a": {"obj": {"component_name": "`+nameA+`", "params": {"chunk_size": 128}}, "upstream": ["begin"], "downstream": ["b"]},- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}- },- "path": ["begin", "a", "b"],- "graph": {"nodes": []}- }- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))- if err != nil {- t.Fatalf("NewPipelineFromDSL (run 2): %v", err)- }- if _, err := pipe2.Run(t.Context(), map[string]any{"name": "doc-dsl-changed"}, nil); err != nil {- t.Fatalf("Run 2: expected fresh run to succeed after DSL edit, got error: %v", err)- }-- // Fresh run re-executed A (and B, which now delegates successfully at n=2).- if mockA.calls != 2 {- t.Fatalf("Run 2: expected A to run again (fresh run), got A.calls=%d", mockA.calls)- }- if termStage.calls != 1 {- t.Fatalf("Run 2: expected B to delegate once, got %d", termStage.calls)- }- if store.deleteCount() < 1 {- t.Fatalf("expected the stale checkpoint to be deleted on DSL change, got deleteCount=%d", store.deleteCount())- }-}--// TestClassifyDSLChange is the table-driven unit test for the mismatch-reason-// classifier used by the warning log. It isolates the "DSL file changed vs-// runtime override changed" decision from the Run/resume plumbing so the-// diagnostic label can be verified directly (the log text itself is not-// asserted by capture).-func TestClassifyDSLChange(t *testing.T) {- const (- dsl = "dslfp"- ovf = "ovrfp"- )- cases := []struct {- name string- storedDsl string- storedOvf string- dslFP string- ovfFP string- wantReason string- }{- {"no change", dsl, ovf, dsl, ovf, ""},- {"dsl file changed", dsl + "x", ovf, dsl, ovf, "DSL/template changed"},- {"override changed", dsl, ovf + "x", dsl, ovf, "runtime override/parser-config changed"},- {"both changed", dsl + "x", ovf + "x", dsl, ovf, "DSL/template changed"},- // Caller only invokes classify when a checkpoint exists; a missing- // fingerprint is treated as "changed" (safe: discard + fresh run).- {"missing stored fingerprint", "", "", dsl, ovf, "DSL/template changed"},- }- for _, c := range cases {- t.Run(c.name, func(t *testing.T) {- got := classifyDSLChange(c.storedDsl, c.storedOvf, c.dslFP, c.ovfFP)- if got != c.wantReason {- t.Fatalf("classifyDSLChange = %q, want %q", got, c.wantReason)- }- })- }-}--// errCheckpointStore fails every Get with getErr and records whether Set was-// ever called. It lets us assert that guardDSLChange does NOT overwrite the-// DSL/override fingerprints after a failed checkpoint lookup (which would mask-// a real stale-checkpoint mismatch on a later resume).-type errCheckpointStore struct {- getErr error- setCalled bool-}--func (s *errCheckpointStore) Get(_ context.Context, _ string) ([]byte, bool, error) {- return nil, false, s.getErr-}-func (s *errCheckpointStore) Set(_ context.Context, _ string, _ []byte) error {- s.setCalled = true- return nil-}-func (s *errCheckpointStore) Delete(_ context.Context, _ string) error { return nil }--// TestGuardDSLChange_CheckpointLookupErrorSkipsOverwrite verifies the-// CodeRabbit finding: when store.Get(cpID) returns an error, guardDSLChange-// bails out entirely and must NOT call Set on the fingerprint keys.-func TestGuardDSLChange_CheckpointLookupErrorSkipsOverwrite(t *testing.T) {- const taskID = "task-lookup-err"- store := &errCheckpointStore{getErr: errors.New("redis temporarily unavailable")}- p := &Pipeline{- taskID: taskID,- rawDSL: []byte(`{"dsl":{"components":{"begin":{"obj":{"component_name":"Begin","params":{}}}}}}`),- }- // tracker is nil-safe (only used on the delete branch, which is skipped).- p.guardDSLChange(t.Context(), store, nil, taskID, map[string]any{"k": "v"})- if store.setCalled {- t.Fatal("guardDSLChange must not overwrite fingerprints after a failed checkpoint lookup")- }-}--// TestPipelineRunResumableOverrideChanged verifies that editing ONLY the-// runtime override_params (same DSL file) between the failed run and the-// resume is detected as a change and forces a fresh run (the checkpoint is-// discarded) — it must NOT resume against stale override state. This is the-// "runtime override/parser-config changed" branch of guardDSLChange.-func TestPipelineRunResumableOverrideChanged(t *testing.T) {- mockA := &mockCanvasStage{output: map[string]any{"a": 1}}- termStage := &oneShotErrStage{mockCanvasStage: mockCanvasStage{output: map[string]any{"b": 2}}}-- const (- nameA = "p.OvfChangedA"- nameB = "p.OvfChangedB"- )- runtime.MustRegister(nameA, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return mockA, nil },- runtime.Metadata{Version: "1.0.0"})- runtime.MustRegister(nameB, runtime.CategoryIngestion,- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },- runtime.Metadata{Version: "1.0.0"})-- const taskID = "task-override-changed"-- store := newMemCheckpointStore()- mr := miniredis.RunT(t)- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})- t.Cleanup(func() { client.Close() })- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)-- dsl := `{- "dsl": {- "components": {- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},- "a": {"obj": {"component_name": "` + nameA + `", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},- "b": {"obj": {"component_name": "` + nameB + `", "params": {}}, "upstream": ["a"]}- },- "path": ["begin", "a", "b"],- "graph": {"nodes": []}- }- }`-- // Run 1: override v1; terminal B errors (oneShotErrStage n=1), leaving a- // checkpoint + interrupt. The DSL file fingerprint is recorded too.- pipe1, err := NewPipelineFromDSL([]byte(dsl), taskID, WithCheckPointStore(store), WithRunTracker(tracker))- if err != nil {- t.Fatalf("NewPipelineFromDSL (run 1): %v", err)- }- if _, err := pipe1.Run(t.Context(), map[string]any{"name": "doc-ovf-changed"}, map[string]any{"k": "v1"}); err == nil {- t.Fatal("Run 1: expected error from simulated crash, got nil")- }- if mockA.calls != 1 {- t.Fatalf("Run 1: expected A to run once, got %d", mockA.calls)- }-- // Run 2: SAME DSL file, but override changed to v2. The guard must detect- // the override change, discard the stale checkpoint, and re-run from- // scratch (A runs again).- pipe2, err := NewPipelineFromDSL([]byte(dsl), taskID, WithCheckPointStore(store), WithRunTracker(tracker))- if err != nil {- t.Fatalf("NewPipelineFromDSL (run 2): %v", err)- }- if _, err := pipe2.Run(t.Context(), map[string]any{"name": "doc-ovf-changed"}, map[string]any{"k": "v2"}); err != nil {- t.Fatalf("Run 2: expected fresh run to succeed after override edit, got error: %v", err)- }- if mockA.calls != 2 {- t.Fatalf("Run 2: expected A to run again (fresh run on override change), got A.calls=%d", mockA.calls)- }- if termStage.calls != 1 {- t.Fatalf("Run 2: expected B to delegate once, got %d", termStage.calls)- }- if store.deleteCount() < 1 {- t.Fatalf("expected the stale checkpoint to be deleted on override change, got deleteCount=%d", store.deleteCount())- }-}--// TestPipelineRun_RequireResumeRejectsWithoutStore verifies that with-// WithRequireResume set and no checkpoint store resolvable (no-// injected store, no global Redis in unit scope), Run must refuse to start-// and return ErrResumeUnavailable — a clear, distinguishable signal — rather-// than silently degrading to a non-resumable runPlain. The reject fires-// before compile, so the DSL does not need a runnable graph.-func TestPipelineRun_RequireResumeRejectsWithoutStore(t *testing.T) {- pipe, err := NewPipelineFromDSL([]byte(`{- "dsl": {- "components": {- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},- "a": {"obj": {"component_name": "p.Docx", "params": {}}, "upstream": ["begin"]}- },- "path": ["begin", "a"],- "graph": {"nodes": []}- }- }`), "task-req-resume", WithRequireResume())- if err != nil {- t.Fatalf("NewPipelineFromDSL: %v", err)- }- _, err = pipe.Run(t.Context(), map[string]any{"name": "doc"}, nil)- if !errors.Is(err, ErrResumeUnavailable) {- t.Fatalf("expected ErrResumeUnavailable, got %v", err)- }-}-// recordingSink captures OnComponentTotal / OnComponentProgress calls so tests// can assert the pipeline forwards progress to the sink instead of writing// the DAO layer directly.@@ -1018,72 +358,6 @@ func TestPipelineRunForwardsProgressToSink(t *testing.T) {}}-// =============================================================================-// cleanupCheckpoint — direct unit test-// =============================================================================--func TestCleanupCheckpoint_DeletesStoreAndClearsTracker(t *testing.T) {- mr := miniredis.RunT(t)- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})- t.Cleanup(func() { client.Close() })-- store := newMemCheckpointStore()- if err := store.Set(t.Context(), "cp-1", []byte("data")); err != nil {- t.Fatalf("store.Set: %v", err)- }- // Fingerprint keys must share the checkpoint's lifecycle on cleanup.- if err := store.Set(t.Context(), "cp-1"+dslKeySuffix, []byte("dslfp")); err != nil {- t.Fatalf("store.Set dsl: %v", err)- }- if err := store.Set(t.Context(), "cp-1"+ovfKeySuffix, []byte("ovrfp")); err != nil {- t.Fatalf("store.Set ovf: %v", err)- }- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)- if err := tracker.AttachInterrupt(t.Context(), "cp-1", "interrupt-1"); err != nil {- t.Fatalf("AttachInterrupt: %v", err)- }-- p := &Pipeline{}- p.cleanupCheckpoint(t.Context(), store, tracker, "cp-1")-- // checkpoint + dsl fingerprint + ovf fingerprint = 3 deletes.- if store.deleteCount() != 3 {- t.Fatalf("expected 3 store deletes (checkpoint + 2 fingerprints), got %d", store.deleteCount())- }- for _, k := range []string{"cp-1", "cp-1" + dslKeySuffix, "cp-1" + ovfKeySuffix} {- if _, found, _ := store.Get(t.Context(), k); found {- t.Fatalf("expected key %q to be deleted", k)- }- }- id, ok, err := tracker.GetInterruptID(t.Context(), "cp-1")- if err != nil {- t.Fatalf("GetInterruptID: %v", err)- }- if ok && id != "" {- t.Fatalf("interrupt id should be cleared, got %q", id)- }-}--func TestCleanupTaskStateDeletesCheckpointAndFingerprints(t *testing.T) {- store := newMemCheckpointStore()- for _, key := range []string{"task-1", "task-1:dsl", "task-1:ovf"} {- if err := store.Set(t.Context(), key, []byte("state")); err != nil {- t.Fatalf("seed %s: %v", key, err)- }- }-- if err := cleanupTaskState(t.Context(), store, nil, "task-1"); err != nil {- t.Fatalf("cleanupTaskState: %v", err)- }- for _, key := range []string{"task-1", "task-1:dsl", "task-1:ovf"} {- if _, ok, err := store.Get(t.Context(), key); err != nil {- t.Fatalf("read %s: %v", key, err)- } else if ok {- t.Fatalf("checkpoint state %s still exists", key)- }- }-}-// =============================================================================// runPlain — tracker integration with miniredis// =============================================================================diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex d48252818e..16f5350501 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -1195,7 +1195,7 @@ func (e *Ingestor) defaultRunDocumentTask(ctx context.Context, ingestionTask *enif ingestionTask.PipelineLogID != nil {pipelineLogID = *ingestionTask.PipelineLogID}- result, err := executor.WithRequireResume().WithProgressSink(newProgressSink(ctx, e.ingestionTaskSvc, pipelineLogID)).Execute(docTaskCtx.Ctx)+ result, err := executor.WithProgressSink(newProgressSink(ctx, e.ingestionTaskSvc, pipelineLogID)).Execute(docTaskCtx.Ctx)if err != nil {return err}diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.goindex bac2bd0ad4..16c3af4e03 100644--- a/internal/ingestion/task/pipeline_executor.go+++ b/internal/ingestion/task/pipeline_executor.go@@ -73,7 +73,6 @@ type PipelineExecutor struct {loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)progressSink pipelinepkg.ProgressSink- requireResume bool // when true, the pipeline run passes WithRequireResume}func validateTaskContext(taskCtx *TaskContext) error {@@ -168,14 +167,6 @@ func (s *PipelineExecutor) WithProgressSink(sink pipelinepkg.ProgressSink) *Pipereturn s}-// WithRequireResume makes the pipeline refuse to start when no checkpoint-// store is resolvable (Redis down or not configured). Production ingestion-// sets this; tests skip it so they can exercise runPlain without Redis.-func (s *PipelineExecutor) WithRequireResume() *PipelineExecutor {- s.requireResume = true- return s-}-func (s *PipelineExecutor) KB() *entity.Knowledgebase { return &s.taskCtx.KB }func (s *PipelineExecutor) Doc() *entity.Document { return &s.taskCtx.Doc }func (s *PipelineExecutor) Tenant() *entity.Tenant { return &s.taskCtx.Tenant }diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.goindex e67b18cd13..2a302978a6 100644--- a/internal/service/document/document_parse.go+++ b/internal/service/document/document_parse.go@@ -58,10 +58,10 @@ func lockDocumentParse(docID string) func() {}}-// purgeTaskStateForCleanup removes resumable checkpoint, tracker, and chunk-// cache state before a task row is deleted. The bounded context makes Redis-// outages and hung clients observable to the caller instead of allowing a-// destructive task/document delete to proceed with stale resume state.+// purgeTaskStateForCleanup removes the per-chunk cache state for a task before+// its row is deleted. The bounded context makes Redis outages and hung clients+// observable to the caller instead of allowing a destructive task/document+// delete to proceed with stale cache state.func (s *DocumentService) purgeTaskStateForCleanup(ctx context.Context, taskID string) error {purgeTaskState := s.purgeTaskStateif purgeTaskState == nil {