Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(go-ingestion): remove checkpoint/resume support (#19924)

xugangqiangcd2e548215b0
diff --git a/internal/ingestion/pipeline/cleanup.go b/internal/ingestion/pipeline/cleanup.go
index 4cb11705f3..40d5541ffa 100644
--- a/internal/ingestion/pipeline/cleanup.go
+++ b/internal/ingestion/pipeline/cleanup.go
@@ -3,55 +3,17 @@ package pipeline
import (
"context"
"errors"
- "fmt"
- "ragflow/internal/agent/canvas"
"ragflow/internal/ingestion/chunkcache"
)
-// cleanupCheckpointState removes the checkpoint and the fingerprints that
-// describe the graph which produced it. It returns storage failures to
-// callers that need to gate a new run on cleanup completion; the pipeline's
-// cancellation path logs those failures and remains best-effort.
-func cleanupCheckpointState(ctx context.Context, store canvas.CheckPointStore, tracker *canvas.RunTracker, checkpointID string) error {
- if checkpointID == "" {
- return errors.New("checkpoint cleanup requires a checkpoint id")
- }
- var cleanupErr error
- if store != nil {
- for _, key := range []string{checkpointID, checkpointID + dslKeySuffix, checkpointID + ovfKeySuffix} {
- if err := store.Delete(ctx, key); err != nil {
- cleanupErr = errors.Join(cleanupErr, fmt.Errorf("delete %s: %w", key, err))
- }
- }
- }
- if tracker != nil {
- if err := tracker.ClearInterruptID(ctx, checkpointID); err != nil {
- cleanupErr = errors.Join(cleanupErr, fmt.Errorf("clear interrupt %s: %w", checkpointID, err))
- }
- }
- return cleanupErr
-}
-
-// cleanupTaskState clears all resumable state associated with a task before
-// its terminal task row is removed for a rerun. Cache entries are purged only
-// after checkpoint cleanup succeeds so a failed cleanup remains retryable.
-func cleanupTaskState(ctx context.Context, store canvas.CheckPointStore, tracker *canvas.RunTracker, taskID string) error {
- if err := cleanupCheckpointState(ctx, store, tracker, taskID); err != nil {
- return err
- }
- if err := chunkcache.PurgeTask(ctx, chunkcache.Client(), taskID); err != nil {
- return fmt.Errorf("purge chunk cache: %w", err)
- }
- return nil
-}
-
-// PurgeTaskState removes checkpoint, tracker, and per-chunk cache state for a
-// task using the process-wide Redis clients.
+// PurgeTaskState removes the per-chunk cache state for a task using the
+// process-wide Kvrocks client. The checkpoint (eino resume) store was removed
+// from the ingestion pipeline, so only the per-chunk LLM/embedding results are
+// purged here; a rerun therefore starts with a cold chunk cache.
func PurgeTaskState(ctx context.Context, taskID string) error {
if taskID == "" {
return errors.New("task state cleanup requires a task id")
}
- p := &Pipeline{taskID: taskID}
- return cleanupTaskState(ctx, p.resolveStore(), p.resolveTracker(), taskID)
+ return chunkcache.PurgeTask(ctx, chunkcache.Client(), taskID)
}
diff --git a/internal/ingestion/pipeline/pipeline.go b/internal/ingestion/pipeline/pipeline.go
index d9aafc648a..43dce51ea3 100644
--- a/internal/ingestion/pipeline/pipeline.go
+++ b/internal/ingestion/pipeline/pipeline.go
@@ -18,9 +18,6 @@ package pipeline
import (
"context"
- "crypto/sha256"
- "encoding/hex"
- "encoding/json"
"errors"
"fmt"
"time"
@@ -30,73 +27,34 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
kvrocks "ragflow/internal/engine/kvrocks"
- "ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/utility"
- "github.com/cloudwego/eino/compose"
"go.uber.org/zap"
)
// Pipeline is a compiled ingestion canvas plus task-scoped metadata.
type Pipeline struct {
- taskID string
- // rawDSL holds the canonical (key-sorted) JSON of the canvas DSL this
- // pipeline was compiled from. It is the basis for the resume-time DSL
- // fingerprint (guardDSLChange): any edit to the template DSL changes
- // these bytes, so a stale checkpoint can be detected and discarded
- // instead of being resumed against an incompatible graph.
- rawDSL []byte
+ taskID string
documentID string // owning document; numeric progress is mirrored back to
// the document table for existing status projections.
canvas *canvas.Canvas
- store canvas.CheckPointStore // optional injected; nil -> resolve at Run
- tracker *canvas.RunTracker // optional injected; nil -> resolve at Run
- // requireResume, when true, makes Run refuse to start if no checkpoint
- // store can be resolved (no injected store AND no global Kvrocks client).
- // Plan §6.a M4: a deployment that cannot persist checkpoints must
- // not silently degrade to a non-resumable run — it must surface a clear,
- // distinguishable error so the caller knows resume is unavailable.
- requireResume bool
- factory runtime.ComponentFactory // optional instance-scoped component factory
- sink ProgressSink // optional progress sink; nil -> drop events (DB-independent)
+ tracker *canvas.RunTracker // optional injected; nil -> resolve at Run
+ factory runtime.ComponentFactory // optional instance-scoped component factory
+ sink ProgressSink // optional progress sink; nil -> drop events (DB-independent)
}
-// ErrResumeUnavailable is returned by Run when WithRequireResume is set but no
-// checkpoint store can be resolved (plan §6.a M4). Callers can test for it with
-// errors.Is to surface a "resume unavailable" condition instead of a generic
-// failure (e.g. refuse to enqueue the task rather than start a non-resumable
-// run).
-var ErrResumeUnavailable = errors.New("resume unavailable: no checkpoint store (Redis down or not configured)")
-
// PipelineOption mutates a Pipeline before Run. Used to inject test doubles
// (in-memory store / miniredis tracker) or dedicated Redis pools.
type PipelineOption func(*Pipeline)
-// WithCheckPointStore injects a checkpoint store. When unset, Run resolves
-// one from the global Kvrocks client (and degrades to a non-resumable run when
-// Redis is unavailable — plan §6.a).
-func WithCheckPointStore(s canvas.CheckPointStore) PipelineOption {
- return func(p *Pipeline) { p.store = s }
-}
-
-// WithRunTracker injects a RunTracker for interrupt-id persistence / crash
-// recovery. When unset, Run resolves one from the global Kvrocks client.
+// WithRunTracker injects a RunTracker for progress mirroring (terminal
+// run-status marks). When unset, Run resolves one from the global Kvrocks
+// client.
func WithRunTracker(t *canvas.RunTracker) PipelineOption {
return func(p *Pipeline) { p.tracker = t }
}
-// WithRequireResume makes Run refuse to start when no checkpoint store can be
-// resolved (no injected store AND no global Kvrocks client). This is plan A: a
-// deployment that cannot persist checkpoints must not silently
-// degrade to a non-resumable run — it must surface a clear, distinguishable
-// error (ErrResumeUnavailable) so the caller knows resume is unavailable.
-// Production ingestion wiring sets this; unit tests leave it off to exercise
-// the non-resumable runPlain fallback.
-func WithRequireResume() PipelineOption {
- return func(p *Pipeline) { p.requireResume = true }
-}
-
// WithDocumentID binds the pipeline's owning document so numeric progress can
// be mirrored back into the document table.
// Pass the empty string to disable the mirror (e.g. headless/test runs where
@@ -153,17 +111,9 @@ func NewPipelineFromDSL(dsl []byte, taskID string, opts ...PipelineOption) (*Pip
if err := ValidatePipeline(cnv); err != nil {
return nil, err
}
- // Capture the canonical canvas DSL bytes for the resume-time DSL
- // fingerprint. json.Marshal sorts map keys, so this is stable across
- // re-decodes of the same logical DSL (formatting-independent).
- rawBytes, err := json.Marshal(canvasDSL)
- if err != nil {
- return nil, fmt.Errorf("pipeline: canonicalize DSL: %w", err)
- }
p := &Pipeline{
taskID: taskID,
canvas: cnv,
- rawDSL: rawBytes,
}
for _, o := range opts {
o(p)
@@ -224,172 +174,10 @@ func cloneMapOrEmpty(m map[string]any) map[string]any {
return out
}
-// defaultCheckpointTTL is the expiry applied to the eino checkpoint payload
-// and the RunTracker hash. A finished run's checkpoint is deleted on success;
-// the TTL only guards against leaks from crashed runs that never clean up.
-//
-// It matches the per-chunk cache TTL (chunkcache.TTL) so a crashed run leaves
-// both its checkpoint and the chunk cache entries to expire on the same horizon.
-var defaultCheckpointTTL = 7 * 24 * time.Hour
-
-// dslKeySuffix / ovfKeySuffix derive the two DSL-fingerprint keys from a
-// checkpoint id. Both live in the same CheckPointStore as the eino payload
-// (under cpID+dslKeySuffix / cpID+ovfKeySuffix) so a resume can compare the
-// current DSL against the one that wrote the checkpoint — without depending
-// on the RunTracker being injected. eino only ever reads/writes the exact
-// cpID, so these sibling keys never collide with eino's checkpoint.
-//
-// The split is deliberate: dslKeySuffix fingerprints the DSL FILE (the full
-// canvas DSL — topology, component params, everything the user edits in the
-// template), while ovfKeySuffix fingerprints only the runtime override_params
-// (Doc.ParserConfig + injected LLM id). This lets the mismatch warning say
-// whether the DSL file changed or the runtime override changed, instead of an
-// ambiguous "params changed".
-const (
- dslKeySuffix = ":dsl"
- ovfKeySuffix = ":ovf"
-)
-
-// dslFileFingerprint returns a stable hash of the FULL canvas DSL this
-// pipeline was compiled from. It captures topology (components, edges, graph
-// grouping) AND the per-component params/defaults — i.e. everything the user
-// can edit in the DSL template file. The raw DSL bytes are key-sorted at
-// decode time, so logical edits (not whitespace) drive the value.
-//
-// It returns an error (instead of a silent fallback) when rawDSL is unset:
-// NewPipelineFromDSL always populates rawDSL, so an empty value means the
-// Pipeline was built through an unexpected path. A fallback that re-encodes
-// p.canvas would produce a different hash (Canvas.NodeParents is json:"-",
-// and struct field order differs from the map form), silently breaking the
-// change-detection contract.
-func (p *Pipeline) dslFileFingerprint() (string, error) {
- if len(p.rawDSL) == 0 {
- return "", fmt.Errorf("rawDSL not set (Pipeline built without NewPipelineFromDSL?)")
- }
- h := sha256.New()
- h.Write(p.rawDSL)
- return hex.EncodeToString(h.Sum(nil)), nil
-}
-
-// overrideFingerprint returns a stable hash of the run-level override_params
-// (Doc.ParserConfig + injected LLM id). It is tracked separately from the DSL
-// file so a change limited to the runtime override is distinguishable (in the
-// warning log) from an edit to the DSL template file.
-func overrideFingerprint(override map[string]any) string {
- h := sha256.New()
- _ = json.NewEncoder(h).Encode(override)
- return hex.EncodeToString(h.Sum(nil))
-}
-
-// classifyDSLChange decides why a stored fingerprint pair no longer matches
-// the current run, for the mismatch warning log. It returns "" when neither
-// changed (no stale checkpoint to discard). A DSL file change takes precedence
-// in the label because an edited template is the more common/expected case;
-// when both changed the label still says DSL. The caller only invokes this
-// when a checkpoint actually exists, so an empty stored fingerprint is treated
-// as "changed" (safe: discard and re-run from scratch).
-func classifyDSLChange(storedDsl, storedOvf, dslFP, ovfFP string) string {
- dslChanged := storedDsl != dslFP
- ovfChanged := storedOvf != ovfFP
- if !dslChanged && !ovfChanged {
- return ""
- }
- if dslChanged {
- return "DSL/template changed"
- }
- return "runtime override/parser-config changed"
-}
-
-// guardDSLChange prevents resuming a checkpoint written by a DIFFERENT DSL
-// than the one currently compiled. eino checkpoints are bound to the graph's
-// node ids / wiring; resuming against a modified graph restores state for
-// nodes that no longer exist (or have different wiring) and makes eino error
-// out. When either the DSL file fingerprint or the runtime override
-// fingerprint differs we drop the stale checkpoint (and the persisted
-// interrupt marker) so the run starts fresh from the entry node. When no
-// checkpoint exists this is simply a fresh run, and we record both
-// fingerprints so a later resume can detect edits. All store writes are
-// best-effort: a write failure must not abort the run.
-func (p *Pipeline) guardDSLChange(ctx context.Context, store canvas.CheckPointStore, tracker *canvas.RunTracker, cpID string, override map[string]any) {
- // Only meaningful when a checkpoint store is wired (the resumable path).
- // The non-resumable runPlain path never calls here, but guard anyway so a
- // future caller cannot nil-panic on store.Get.
- if store == nil {
- return
- }
- dslKey := cpID + dslKeySuffix
- ovfKey := cpID + ovfKeySuffix
-
- dslFP, err := p.dslFileFingerprint()
- if err != nil {
- // Without a DSL fingerprint we cannot detect edits. Skip the guard
- // rather than risk an empty hash that silently mismatches every run.
- // The run still proceeds (best-effort); resume safety is unchanged
- // only because rawDSL is always set via NewPipelineFromDSL.
- common.Error(fmt.Sprintf("pipeline: skip DSL-change guard for %s: %v", p.taskID, err), err)
- return
- }
- ovfFP := overrideFingerprint(override)
-
- // A checkpoint from a previous run exists → this is a resume candidate.
- // A read error means we cannot trust the existence check, so bail out
- // entirely: do NOT overwrite cpID:dsl / cpID:ovf. Writing current
- // fingerprints after a failed lookup could mask a real stale-checkpoint
- // mismatch on a later resume (the old checkpoint would survive, but its
- // fingerprint would be overwritten to match the new DSL).
- _, found, cpErr := store.Get(ctx, cpID)
- if cpErr != nil {
- common.Error(fmt.Sprintf("pipeline: lookup checkpoint for %s failed, skip DSL-change guard: %v", p.taskID, cpErr), cpErr)
- return
- }
- if found {
- storedDsl, dOk, dErr := store.Get(ctx, dslKey)
- storedOvf, oOk, oErr := store.Get(ctx, ovfKey)
- if dErr != nil || oErr != nil {
- common.Error(fmt.Sprintf("pipeline: read DSL fingerprints for %s failed, skip DSL-change guard: %v", p.taskID, coalesceErr(dErr, oErr)), coalesceErr(dErr, oErr))
- return
- }
- reason := classifyDSLChange(orEmpty(dOk, storedDsl), orEmpty(oOk, storedOvf), dslFP, ovfFP)
- if reason != "" {
- common.Warn(fmt.Sprintf("pipeline: DSL for task %s changed since checkpoint was written (%s); discarding stale checkpoint and re-running from scratch", p.taskID, reason))
- if err := store.Delete(ctx, cpID); err != nil {
- common.Error(fmt.Sprintf("pipeline: delete stale checkpoint for %s failed: %v", p.taskID, err), err)
- }
- if tracker != nil {
- if err := tracker.ClearInterruptID(ctx, cpID); err != nil {
- common.Error(fmt.Sprintf("pipeline: clear interrupt id for %s failed: %v", p.taskID, err), err)
- }
- }
- }
- }
- // Record the fingerprints of the DSL + override that produced this run.
- if err := store.Set(ctx, dslKey, []byte(dslFP)); err != nil {
- common.Error(fmt.Sprintf("pipeline: persist DSL fingerprint for %s failed: %v", p.taskID, err), err)
- }
- if err := store.Set(ctx, ovfKey, []byte(ovfFP)); err != nil {
- common.Error(fmt.Sprintf("pipeline: persist override fingerprint for %s failed: %v", p.taskID, err), err)
- }
-}
-
-// orEmpty returns the byte slice when ok, else an empty string, so a missing
-// stored fingerprint is treated as "" (classifyDSLChange then sees a change).
-func orEmpty(ok bool, b []byte) string {
- if !ok {
- return ""
- }
- return string(b)
-}
-
-// coalesceErr returns the first non-nil error, so a single message can report
-// whichever of two reads failed without dereferencing nil.
-func coalesceErr(errs ...error) error {
- for _, e := range errs {
- if e != nil {
- return e
- }
- }
- return nil
-}
+// defaultTrackerTTL is the expiry applied to the RunTracker run-status hash.
+// It matches the per-chunk cache TTL (chunkcache.TTL) so both expire on the
+// same horizon.
+var defaultTrackerTTL = 7 * 24 * time.Hour
// Run executes the full ingestion graph described by the canonical DSL.
// There is no pipeline-layer partial resume entry point: execution always
@@ -414,43 +202,15 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam
compileCtx = canvas.WithComponentFactory(compileCtx, p.factory)
}
- // Resolve the checkpoint store + run tracker. Resume (interrupt-after
- // non-terminal node) requires a checkpoint store to persist the resume
- // point; without one we fall back to a single non-resumable Invoke
- // (plan §6.a degrade — progress stays observable, the run just cannot
- // pause/resume across nodes).
- store := p.resolveStore()
+ // Resolve the run tracker for progress mirroring (terminal
+ // MarkSucceeded/MarkFailed/MarkCancelled marks). Resume-from-checkpoint is
+ // intentionally not supported: the pipeline always runs the full graph
+ // from the entry node, and component-level replay decisions (e.g. serving
+ // prior LLM/embedding results from the per-chunk cache) belong to the
+ // components themselves.
tracker := p.resolveTracker()
- // M4 (plan §6.a): refuse to start when resume is required but no
- // checkpoint store is resolvable. A Redis-less deployment must not pretend
- // the task is resumable; it must report the gap clearly so the caller can
- // refuse to enqueue the task instead of silently running a non-resumable
- // run (which would violate "re-run same task, completed components not
- // redone").
- if p.requireResume && store == nil {
- return nil, fmt.Errorf("pipeline: Run: %w", ErrResumeUnavailable)
- }
- resumable := store != nil
-
var compileOpts []canvas.CompileOption
- if resumable {
- compileOpts = append(compileOpts,
- canvas.WithCheckPointStore(store),
- canvas.WithCheckPointID(p.taskID),
- )
- // Single interrupt point: immediately after the Parser component. The
- // Extractor / Tokenizer stages do NOT take a checkpoint of their own —
- // a resume re-runs them, but the per-chunk cache (chunkcache) absorbs
- // the cost by serving prior LLM/embedding results. Interruption after
- // every non-terminal component was dropped on purpose to avoid one
- // needless ResumeWithData round per stage.
- if parserCpnID := ExtractParserCpnID(p.rawDSL, component.ComponentNameParser); parserCpnID != "" {
- compileOpts = append(compileOpts, canvas.WithInterruptAfter([]string{parserCpnID}))
- } else {
- compileOpts = append(compileOpts, canvas.WithInterruptAfterNonTerminalCpn())
- }
- }
// Run-level setups (keyed by cpnID) override the DSL-baked component
// setups at compile time (higher priority; see canvas.WithOverrideParams).
if overrideParams != nil {
@@ -495,50 +255,24 @@ func (p *Pipeline) Run(ctx context.Context, inputs map[string]any, overrideParam
// no-op, leaving orphaned cache entries until TTL expiry.
globals.SetTaskID(runCtx, p.taskID)
- if !resumable {
- return p.runPlain(runCtx, current, compiled, tracker, runState)
- }
-
- // Resumable path: detect DSL / override edits since the checkpoint was
- // written and discard a stale checkpoint before resuming (see guardDSLChange).
- p.guardDSLChange(ctx, store, tracker, p.taskID, overrideParams)
-
- // Resumable path: record the run, then loop Invoke until the graph
- // completes or a non-resumable error surfaces.
- if tracker != nil {
- if err = tracker.Start(ctx, p.taskID, "", "", ""); err != nil {
- common.Error(fmt.Sprintf("pipeline: RunTracker.Start for task %s failed: %v", p.taskID, err), err)
- }
- }
- return p.runResumable(ctx, runCtx, current, compiled, store, tracker, runState)
-}
-
-// resolveStore returns the injected store, or a Kvrocks-backed one when the
-// global Kvrocks client is available. Returns nil (degraded, non-resumable)
-// when neither is present.
-func (p *Pipeline) resolveStore() canvas.CheckPointStore {
- if p.store != nil {
- return p.store
- }
- if kvrocks.Get() != nil {
- return canvas.NewKvrocksCheckPointStore(defaultCheckpointTTL)
- }
- return nil
+ return p.runPlain(runCtx, current, compiled, tracker, runState)
}
-// resolveTracker mirrors resolveStore for the RunTracker.
+// resolveTracker resolves the RunTracker used for progress mirroring. When an
+// instance-scoped tracker was injected it wins; otherwise one is created on
+// the global Kvrocks client when available.
func (p *Pipeline) resolveTracker() *canvas.RunTracker {
if p.tracker != nil {
return p.tracker
}
if kvrocks.Get() != nil {
- return canvas.NewRunTracker(defaultCheckpointTTL)
+ return canvas.NewRunTracker(defaultTrackerTTL)
}
return nil
}
-// runPlain executes a single Invoke with no checkpoint/resume. Used when no
-// checkpoint store is available; progress is still recorded via the sink.
+// runPlain executes the full ingestion graph with a single Invoke and records
+// the terminal run status via the sink/RunTracker when available.
func (p *Pipeline) runPlain(runCtx context.Context, current map[string]any, compiled *canvas.CompiledCanvas, tracker *canvas.RunTracker, runState *canvas.CanvasState) (map[string]any, error) {
// Terminal tracker writes must survive a run ctx that gets cancelled
// (the common failure/cancel path). Derive a fresh detached ctx per
@@ -573,113 +307,8 @@ func (p *Pipeline) runPlain(runCtx context.Context, current map[string]any, comp
return finalizeResult(current, out, runState), nil
}
-// runResumable drives the graph with eino's interrupt-after-node + resume
-// loop (plan §8 step 3). Every non-terminal-node pause is auto-resumed with
-// nil data (ingestion resume needs no user input). The loop's TOP reads any
-// persisted interrupt id — from the RunTracker (cross-process crash
-// recovery) or an in-process fallback — and resumes; the BOTTOM only persists
-// the id, never inline-resumes (avoids double-resuming one ctx, plan §4.2
-// 建议2).
-func (p *Pipeline) runResumable(ctx context.Context, runCtx context.Context, current map[string]any, compiled *canvas.CompiledCanvas, store canvas.CheckPointStore, tracker *canvas.RunTracker, runState *canvas.CanvasState) (map[string]any, error) {
- cpID := compiled.CheckPointID
- var localInterruptID string // in-process resume fallback when tracker is nil
-
- const maxResumeRounds = 1000
- for round := 0; round < maxResumeRounds; round++ {
- // TOP: recover the pending interrupt (crash recovery or in-process).
- resumeID := ""
- if tracker != nil {
- if id, ok, _ := tracker.GetInterruptID(ctx, cpID); ok && id != "" {
- resumeID = id
- }
- }
- if resumeID == "" {
- resumeID = localInterruptID
- }
- if resumeID != "" {
- // Mark the resume target. Keep passing the full original input
- // instead of nil: when eino loads the checkpoint it ignores the
- // passed input entirely (restoreCheckPointState only restores
- // channels/state, restoreTasks replays cp.Inputs), and when the
- // checkpoint is missing it silently starts a FRESH run from the
- // entry node — so nil input would re-execute the source node
- // (File) without doc_id / file[0].name and fail with "inputs
- // missing". Passing `current` covers both cases with no
- // check-then-use race.
- runCtx = compose.ResumeWithData(runCtx, resumeID, nil)
- }
-
- out, invokeErr := compiled.Workflow.Invoke(runCtx, current, compose.WithCheckPointID(cpID))
- if invokeErr == nil {
- // Terminal-state writes must survive a ctx cancelled around the
- // Invoke boundary. Derive a fresh detached ctx here — bounded by
- // a short timeout so a hung Redis cannot stall completion — so
- // the 5s window covers only the cleanup, not the whole run
- // (mirrors markStopped/markFailed in ingestion_service.go).
- stateCtx, stateCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
- defer stateCancel()
- if tracker != nil {
- utility.BestEffort(fmt.Sprintf("ClearInterruptID for %s", p.taskID), func() error { return tracker.ClearInterruptID(stateCtx, cpID) })
- utility.BestEffort(fmt.Sprintf("MarkSucceeded for %s", p.taskID), func() error { return tracker.MarkSucceeded(stateCtx, cpID) })
- }
- if store != nil {
- utility.BestEffort(fmt.Sprintf("delete checkpoint for %s", p.taskID), func() error { return store.Delete(stateCtx, cpID) })
- utility.BestEffort(fmt.Sprintf("delete DSL fingerprint for %s", p.taskID), func() error { return store.Delete(stateCtx, cpID+dslKeySuffix) })
- utility.BestEffort(fmt.Sprintf("delete override fingerprint for %s", p.taskID), func() error { return store.Delete(stateCtx, cpID+ovfKeySuffix) })
- }
- return finalizeResult(current, out, runState), nil
- }
-
- // Cancellation (plan §4.3.b): wipe the checkpoint and mark cancelled.
- if errors.Is(ctx.Err(), context.Canceled) || errors.Is(ctx.Err(), context.DeadlineExceeded) {
- // The run ctx is already cancelled, so Redis writes with it would
- // fail immediately. Derive a fresh detached ctx so cleanup +
- // MarkCancelled actually land; the 5s window covers the cleanup
- // only, regardless of how long the run itself took.
- cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
- defer cleanupCancel()
- p.cleanupCheckpoint(cleanupCtx, store, tracker, cpID)
- if tracker != nil {
- utility.BestEffort(fmt.Sprintf("MarkCancelled for %s", p.taskID), func() error { return tracker.MarkCancelled(cleanupCtx, cpID) })
- }
- return current, fmt.Errorf("pipeline: run cancelled: %w", ctx.Err())
- }
-
- if !canvas.IsInterruptError(invokeErr) {
- // Same detached-ctx guarantee as the success/cancel paths: the
- // run ctx may be cancelled by the time the failure surfaces.
- failCtx, failCancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
- defer failCancel()
- if tracker != nil {
- utility.BestEffort(fmt.Sprintf("MarkFailed for %s", p.taskID), func() error { return tracker.MarkFailed(failCtx, cpID, invokeErr.Error()) })
- }
- return current, fmt.Errorf("pipeline: run canvas workflow: %w", invokeErr)
- }
-
- // Paused at a non-terminal node: persist for crash recovery, then
- // resume on the next loop iteration's TOP.
- ctxs := canvas.ExtractInterruptContexts(invokeErr)
- id := canvas.FirstInterruptID(ctxs)
- localInterruptID = id
- if tracker != nil {
- if err := tracker.AttachInterrupt(ctx, cpID, id); err != nil {
- common.Error(fmt.Sprintf("pipeline: AttachInterrupt for task %s failed: %v", p.taskID, err), err)
- }
- }
- }
- return current, fmt.Errorf("pipeline: run exceeded max resume rounds (%d) for task %s", maxResumeRounds, p.taskID)
-}
-
-// cleanupCheckpoint wipes the eino checkpoint payload and the persisted
-// interrupt id (plan §4.3.b cancelled path).
-func (p *Pipeline) cleanupCheckpoint(ctx context.Context, store canvas.CheckPointStore, tracker *canvas.RunTracker, cpID string) {
- if err := cleanupCheckpointState(ctx, store, tracker, cpID); err != nil {
- common.Error(fmt.Sprintf("pipeline: cleanup checkpoint %s failed: %v", cpID, err), err)
- }
-}
-
// finalizeResult merges the graph output into the input map and attaches the
-// canvas state snapshot — the shared success payload for both run paths.
+// canvas state snapshot as the success payload.
func finalizeResult(current, out map[string]any, runState *canvas.CanvasState) map[string]any {
if out == nil {
current["state"] = runState.Snapshot()
diff --git a/internal/ingestion/pipeline/pipeline_test.go b/internal/ingestion/pipeline/pipeline_test.go
index 8af84ed3ec..57b7c50f42 100644
--- a/internal/ingestion/pipeline/pipeline_test.go
+++ b/internal/ingestion/pipeline/pipeline_test.go
@@ -18,7 +18,6 @@ package pipeline
import (
"context"
- "errors"
"fmt"
"strings"
"sync"
@@ -51,25 +50,6 @@ func (m *mockCanvasStage) Invoke(_ context.Context, _ *gorm.DB, inputs map[strin
func (m *mockCanvasStage) Inputs() map[string]string { return map[string]string{"name": "string"} }
func (m *mockCanvasStage) Outputs() map[string]string { return map[string]string{"output": "any"} }
-// oneShotErrStage errors on the first Invoke (simulating a component crash
-// mid-run), then delegates to the embedded mock on subsequent calls. Used to
-// test that a second pipeline Run on the same taskID resumes past non-terminal
-// checkpoints instead of re-executing completed components.
-type oneShotErrStage struct {
- mockCanvasStage
- n int
-}
-
-func (s *oneShotErrStage) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
- s.n++
- if s.n == 1 {
- return nil, errors.New("simulated crash")
- }
- return s.mockCanvasStage.Invoke(ctx, db, inputs)
-}
-func (s *oneShotErrStage) Inputs() map[string]string { return s.mockCanvasStage.Inputs() }
-func (s *oneShotErrStage) Outputs() map[string]string { return s.mockCanvasStage.Outputs() }
-
func TestPipelineRunHappyPath(t *testing.T) {
stageA := &mockCanvasStage{output: map[string]any{"a": 1}}
stageB := &mockCanvasStage{output: map[string]any{"b": 2}}
@@ -189,48 +169,6 @@ func (s *factorySentinelStage) Invoke(_ context.Context, _ *gorm.DB, inputs map[
return out, nil
}
-// memCheckpointStore is a thread-safe in-memory canvas.CheckPointStore used
-// to exercise the resumable run path without Redis.
-type memCheckpointStore struct {
- mu sync.Mutex
- data map[string][]byte
- deleted int // number of times Delete was called
-}
-
-func newMemCheckpointStore() *memCheckpointStore {
- return &memCheckpointStore{data: map[string][]byte{}}
-}
-
-func (s *memCheckpointStore) Get(_ context.Context, id string) ([]byte, bool, error) {
- s.mu.Lock()
- defer s.mu.Unlock()
- v, ok := s.data[id]
- return v, ok, nil
-}
-
-func (s *memCheckpointStore) Set(_ context.Context, id string, payload []byte) error {
- s.mu.Lock()
- defer s.mu.Unlock()
- cp := make([]byte, len(payload))
- copy(cp, payload)
- s.data[id] = cp
- return nil
-}
-
-func (s *memCheckpointStore) Delete(_ context.Context, id string) error {
- s.mu.Lock()
- defer s.mu.Unlock()
- delete(s.data, id)
- s.deleted++
- return nil
-}
-
-func (s *memCheckpointStore) deleteCount() int {
- s.mu.Lock()
- defer s.mu.Unlock()
- return s.deleted
-}
-
// TestPipelineRun_InstanceFactoryOverridesDefaultFactory verifies that a
// pipeline-scoped component factory can provide task-specific components.
func TestPipelineRun_InstanceFactoryOverridesDefaultFactory(t *testing.T) {
@@ -336,604 +274,6 @@ func TestPipelineRun_TaskScopedFactoriesDoNotLeakAcrossConcurrentPipelines(t *te
}
}
-func TestPipelineRunResumableAutoResumes(t *testing.T) {
- stageA := &mockCanvasStage{output: map[string]any{"a": 1}}
- stageB := &mockCanvasStage{output: map[string]any{"b": 2}}
-
- const (
- nameA = "p.ResumeStageA"
- nameB = "p.ResumeStageB"
- )
- runtime.MustRegister(nameA, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return stageA, nil },
- runtime.Metadata{Version: "1.0.0"})
- runtime.MustRegister(nameB, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return stageB, nil },
- runtime.Metadata{Version: "1.0.0"})
-
- pipe, err := NewPipelineFromDSL([]byte(`{
- "dsl": {
- "components": {
- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},
- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}
- },
- "path": ["begin", "a", "b"],
- "graph": {"nodes": []}
- }
- }`), "task-resume", WithCheckPointStore(newMemCheckpointStore()))
- if err != nil {
- t.Fatalf("NewPipelineFromDSL: %v", err)
- }
-
- out, err := pipe.Run(t.Context(), map[string]any{"name": "doc-resume"}, nil)
- if err != nil {
- t.Fatalf("Run: %v", err)
- }
- if !stageA.called || !stageB.called {
- t.Fatalf("expected both stages to run, got A=%v B=%v", stageA.called, stageB.called)
- }
- // No re-run on resume: each node must execute exactly once.
- if stageA.calls != 1 || stageB.calls != 1 {
- t.Fatalf("expected each stage to run exactly once, got A.calls=%d B.calls=%d", stageA.calls, stageB.calls)
- }
- if out == nil {
- t.Fatal("expected non-nil output")
- }
-}
-
-// TestPipelineRunResumableCrossRunResume validates crash-recovery resume
-// across two Run calls: when the terminal component errors mid-run (simulated
-// crash), non-terminal checkpoints + interrupt state persist. The second Run
-// on the same taskID resumes past completed non-terminal components instead of
-// re-executing them. A non-terminal stage (A) runs exactly once across both runs; the
-// terminal stage (B, a oneShotErrStage) errors on run 1 and succeeds on run 2.
-func TestPipelineRunResumableCrossRunResume(t *testing.T) {
- mockA := &mockCanvasStage{output: map[string]any{"a": 1}}
- mockB := &mockCanvasStage{output: map[string]any{"b": 2}}
- termStage := &oneShotErrStage{mockCanvasStage: *mockB}
-
- const (
- nameA = "p.XRunStageA"
- nameB = "p.XRunStageB"
- )
- runtime.MustRegister(nameA, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return mockA, nil },
- runtime.Metadata{Version: "1.0.0"})
- runtime.MustRegister(nameB, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },
- runtime.Metadata{Version: "1.0.0"})
-
- store := newMemCheckpointStore()
- mr := miniredis.RunT(t)
- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
- t.Cleanup(func() { client.Close() })
- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)
-
- const taskID = "task-cross-run-resume"
- pipe, err := NewPipelineFromDSL([]byte(`{
- "dsl": {
- "components": {
- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},
- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}
- },
- "path": ["begin", "a", "b"],
- "graph": {"nodes": []}
- }
- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))
- if err != nil {
- t.Fatalf("NewPipelineFromDSL: %v", err)
- }
-
- // Run 1: terminal B errors (oneShotErrStage n=1), simulating a crash.
- // Non-terminal A's checkpoint + interrupt persist because the error path
- // does not call ClearInterruptID or store.Delete.
- _, err = pipe.Run(t.Context(), map[string]any{"name": "doc-cross-run"}, nil)
- if err == nil {
- t.Fatal("Run 1: expected error from simulated crash, got nil")
- }
- if mockA.calls != 1 {
- t.Fatalf("Run 1: expected A to run once, got %d", mockA.calls)
- }
- // oneShotErrStage did not delegate to its embedded mock on the first call.
- if termStage.calls != 0 {
- t.Fatalf("Run 1: expected B (embedded mock) calls=0 (error before delegate), got %d", termStage.calls)
- }
-
- // Run 2: resume from after A via tracker.GetInterruptID. A is skipped;
- // B's oneShotErrStage (n=2) delegates to its embedded mock successfully.
- _, err = pipe.Run(t.Context(), map[string]any{"name": "doc-cross-run"}, nil)
- if err != nil {
- t.Fatalf("Run 2: expected recovery success, got error: %v", err)
- }
- if mockA.calls != 1 {
- t.Fatalf("Run 2: expected A to still have calls=1 (was skipped by resume), got %d", mockA.calls)
- }
- if termStage.calls != 1 {
- t.Fatalf("Run 2: expected B (embedded mock) calls=1 (delegated once), got %d", termStage.calls)
- }
-}
-
-// docIDGuardStage mirrors the File component's contract: it requires
-// doc_id (or an explicit file descriptor) in its input and fails otherwise.
-// It lets the resume tests observe whether the graph was re-entered with the
-// original run input or with a nil input (the "inputs missing" failure).
-type docIDGuardStage struct {
- mockCanvasStage
- mu sync.Mutex
- guardCalls int
- missingInputs int
-}
-
-func (m *docIDGuardStage) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
- m.mu.Lock()
- m.guardCalls++
- m.mu.Unlock()
- if inputs == nil || inputs["doc_id"] == nil {
- m.mu.Lock()
- m.missingInputs++
- m.mu.Unlock()
- return nil, errors.New("file: inputs missing doc_id or file[0].name")
- }
- return m.mockCanvasStage.Invoke(ctx, db, inputs)
-}
-
-func (m *docIDGuardStage) snapshot() (calls, missing int) {
- m.mu.Lock()
- defer m.mu.Unlock()
- return m.guardCalls, m.missingInputs
-}
-
-// TestPipelineRunResumableOrphanInterruptIDWithoutCheckpoint covers the
-// cancel-race state: the interrupt id survives in the RunTracker but the
-// checkpoint payload is gone. Before the fix, runResumable treated the
-// orphaned id as "resume" and invoked the graph with nil input, which
-// re-entered the source node (docIDGuardStage) and failed with
-// "inputs missing doc_id". The fix keeps passing the full original input on
-// resume rounds, so the graph starts fresh with doc_id intact instead of
-// re-entering the source node with nil input.
-func TestPipelineRunResumableOrphanInterruptIDWithoutCheckpoint(t *testing.T) {
- guard := &docIDGuardStage{mockCanvasStage: mockCanvasStage{output: map[string]any{"a": 1}}}
- termStage := &oneShotErrStage{mockCanvasStage: mockCanvasStage{output: map[string]any{"b": 2}}}
-
- const (
- nameA = "p.OrphanA"
- nameB = "p.OrphanB"
- )
- runtime.MustRegister(nameA, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return guard, nil },
- runtime.Metadata{Version: "1.0.0"})
- runtime.MustRegister(nameB, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },
- runtime.Metadata{Version: "1.0.0"})
-
- const taskID = "task-orphan-interrupt"
-
- store := newMemCheckpointStore()
- mr := miniredis.RunT(t)
- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
- t.Cleanup(func() {
- if err := client.Close(); err != nil {
- t.Errorf("client.Close: %v", err)
- }
- })
- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)
-
- newPipe := func() *Pipeline {
- pipe, err := NewPipelineFromDSL([]byte(`{
- "dsl": {
- "components": {
- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},
- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}
- },
- "path": ["begin", "a", "b"],
- "graph": {"nodes": []}
- }
- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))
- if err != nil {
- t.Fatalf("NewPipelineFromDSL: %v", err)
- }
- return pipe
- }
-
- // Run 1: terminal B errors, leaving a checkpoint + interrupt id.
- pipe1 := newPipe()
- _, err := pipe1.Run(context.Background(), map[string]any{"doc_id": "d1"}, nil)
- if err == nil {
- t.Fatal("Run 1: expected error from simulated crash, got nil")
- }
- if calls, missing := guard.snapshot(); calls != 1 || missing != 0 {
- t.Fatalf("Run 1: guard.calls=%d missing=%d, want 1/0", calls, missing)
- }
-
- // Simulate the cancel-race: the checkpoint payload vanishes but the
- // interrupt id stays behind (cleanup used a cancelled ctx and failed).
- if err := store.Delete(context.Background(), taskID); err != nil {
- t.Fatalf("store.Delete: %v", err)
- }
- if _, ok, _ := tracker.GetInterruptID(context.Background(), taskID); !ok {
- t.Fatal("precondition: interrupt id must still be present after checkpoint deletion")
- }
-
- // Run 2: must start fresh with the full input preserved (no nil-input
- // re-entry). The guard stage must see doc_id on every invocation
- // (no "inputs missing").
- pipe2 := newPipe()
- _, err = pipe2.Run(context.Background(), map[string]any{"doc_id": "d1"}, nil)
- if err != nil {
- t.Fatalf("Run 2: expected success after checkpoint loss, got: %v", err)
- }
- calls, missing := guard.snapshot()
- if missing != 0 {
- t.Fatalf("Run 2: source stage ran %d times with missing doc_id, want 0 (orphan interrupt id must not nil the input)", missing)
- }
- if calls != 2 {
- t.Fatalf("Run 2: source stage calls=%d, want 2 (fresh run after checkpoint loss)", calls)
- }
-}
-
-// longRunCancelStage simulates a long-running component (e.g. a PDF parser):
-// it blocks until the run ctx is cancelled, then returns a cancellation error
-// so runResumable takes the cancel branch. The started channel lets the test
-// coordinate the cancel so it lands AFTER the entry-derived stateCtx's 5s
-// window would have expired — reproducing the "run took longer than the
-// detached-cleanup budget" failure mode.
-type longRunCancelStage struct {
- started chan struct{}
-}
-
-func (m *longRunCancelStage) Invoke(ctx context.Context, _ *gorm.DB, inputs map[string]any) (map[string]any, error) {
- if m.started != nil {
- close(m.started)
- }
- <-ctx.Done()
- return nil, ctx.Err()
-}
-func (m *longRunCancelStage) Inputs() map[string]string { return map[string]string{"name": "string"} }
-func (m *longRunCancelStage) Outputs() map[string]string { return map[string]string{"output": "any"} }
-
-// TestPipelineRunResumableCancelCleanupAfterLongRun covers the detached-ctx
-// timing defect: the previous "unified" stateCtx was derived once at the top
-// of runResumable with a 5s timeout, so any run lasting longer than 5s
-// reached the cancel branch with an already-expired ctx and the cleanup
-// (checkpoint delete + ClearInterruptID + MarkCancelled) silently failed —
-// reintroducing the exact leak this PR fixes. Each terminal branch must
-// derive its own detached ctx at the point of cleanup.
-func TestPipelineRunResumableCancelCleanupAfterLongRun(t *testing.T) {
- started := make(chan struct{})
- longStage := &longRunCancelStage{started: started}
- termStage := &mockCanvasStage{output: map[string]any{"b": 2}}
-
- const (
- nameA = "p.LongCancelA"
- nameB = "p.LongCancelB"
- )
- runtime.MustRegister(nameA, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return longStage, nil },
- runtime.Metadata{Version: "1.0.0"})
- runtime.MustRegister(nameB, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },
- runtime.Metadata{Version: "1.0.0"})
-
- const taskID = "task-long-run-cancel"
-
- store := newMemCheckpointStore()
- mr := miniredis.RunT(t)
- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
- t.Cleanup(func() {
- if err := client.Close(); err != nil {
- t.Errorf("client.Close: %v", err)
- }
- })
- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)
-
- pipe, err := NewPipelineFromDSL([]byte(`{
- "dsl": {
- "components": {
- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},
- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}
- },
- "path": ["begin", "a", "b"],
- "graph": {"nodes": []}
- }
- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))
- if err != nil {
- t.Fatalf("NewPipelineFromDSL: %v", err)
- }
-
- runCtx, cancel := context.WithCancel(context.Background())
- go func() {
- <-started
- // Simulate a long parse: cancel only after the entry-derived 5s
- // cleanup window would have expired. 6s > 5s. Keep the test fast by
- // timing the sleep from component start rather than from Run entry.
- time.Sleep(6 * time.Second)
- cancel()
- }()
-
- _, err = pipe.Run(runCtx, map[string]any{"doc_id": "d1"}, nil)
- if err == nil {
- t.Fatal("expected cancellation error, got nil")
- }
-
- // The cancel branch must have cleaned up with a fresh detached ctx.
- if _, ok, _ := store.Get(context.Background(), taskID); ok {
- t.Error("checkpoint was not deleted after cancellation (detached-ctx cleanup failed)")
- }
- if _, ok, _ := tracker.GetInterruptID(context.Background(), taskID); ok {
- t.Error("interrupt id was not cleared after cancellation (detached-ctx cleanup failed)")
- }
- fields, err := tracker.Get(context.Background(), taskID)
- if err != nil {
- t.Fatalf("tracker.Get: %v", err)
- }
- if status := fields["status"]; status != "3" {
- t.Errorf("tracker status = %q, want %q (cancelled)", status, "3")
- }
-}
-
-// TestPipelineRunResumableDSLChanged discards a stale checkpoint when the DSL
-// is edited between the failed run and the resume. Run 1 fails on the terminal
-// stage, persisting a checkpoint keyed by taskID. Run 2 re-runs the SAME
-// taskID but with a modified DSL (a param edit, enough to change the
-// fingerprint). The guard must detect the mismatch, discard the checkpoint,
-// and start fresh — so every stage re-executes (A runs on both runs) instead
-// of erroring while resuming an incompatible graph (the original bug).
-func TestPipelineRunResumableDSLChanged(t *testing.T) {
- mockA := &mockCanvasStage{output: map[string]any{"a": 1}}
- termStage := &oneShotErrStage{mockCanvasStage: mockCanvasStage{output: map[string]any{"b": 2}}}
-
- const (
- nameA = "p.DSLChangedA"
- nameB = "p.DSLChangedB"
- )
- runtime.MustRegister(nameA, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return mockA, nil },
- runtime.Metadata{Version: "1.0.0"})
- runtime.MustRegister(nameB, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },
- runtime.Metadata{Version: "1.0.0"})
-
- const taskID = "task-dsl-changed"
-
- // Shared store + tracker across both runs (same taskID).
- store := newMemCheckpointStore()
- mr := miniredis.RunT(t)
- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
- t.Cleanup(func() { client.Close() })
- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)
-
- // Run 1: original DSL; terminal B errors (oneShotErrStage n=1), leaving a
- // checkpoint + interrupt for the original DSL fingerprint.
- pipe1, err := NewPipelineFromDSL([]byte(`{
- "dsl": {
- "components": {
- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
- "a": {"obj": {"component_name": "`+nameA+`", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},
- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}
- },
- "path": ["begin", "a", "b"],
- "graph": {"nodes": []}
- }
- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))
- if err != nil {
- t.Fatalf("NewPipelineFromDSL (run 1): %v", err)
- }
- if _, err := pipe1.Run(t.Context(), map[string]any{"name": "doc-dsl-changed"}, nil); err == nil {
- t.Fatal("Run 1: expected error from simulated crash, got nil")
- }
- if mockA.calls != 1 {
- t.Fatalf("Run 1: expected A to run once, got %d", mockA.calls)
- }
-
- // Run 2: SAME taskID, but a modified DSL (param added to "a" → different
- // fingerprint). Without the guard this would resume the stale checkpoint
- // against an incompatible graph and error; with the guard it discards and
- // re-runs from scratch.
- pipe2, err := NewPipelineFromDSL([]byte(`{
- "dsl": {
- "components": {
- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
- "a": {"obj": {"component_name": "`+nameA+`", "params": {"chunk_size": 128}}, "upstream": ["begin"], "downstream": ["b"]},
- "b": {"obj": {"component_name": "`+nameB+`", "params": {}}, "upstream": ["a"]}
- },
- "path": ["begin", "a", "b"],
- "graph": {"nodes": []}
- }
- }`), taskID, WithCheckPointStore(store), WithRunTracker(tracker))
- if err != nil {
- t.Fatalf("NewPipelineFromDSL (run 2): %v", err)
- }
- if _, err := pipe2.Run(t.Context(), map[string]any{"name": "doc-dsl-changed"}, nil); err != nil {
- t.Fatalf("Run 2: expected fresh run to succeed after DSL edit, got error: %v", err)
- }
-
- // Fresh run re-executed A (and B, which now delegates successfully at n=2).
- if mockA.calls != 2 {
- t.Fatalf("Run 2: expected A to run again (fresh run), got A.calls=%d", mockA.calls)
- }
- if termStage.calls != 1 {
- t.Fatalf("Run 2: expected B to delegate once, got %d", termStage.calls)
- }
- if store.deleteCount() < 1 {
- t.Fatalf("expected the stale checkpoint to be deleted on DSL change, got deleteCount=%d", store.deleteCount())
- }
-}
-
-// TestClassifyDSLChange is the table-driven unit test for the mismatch-reason
-// classifier used by the warning log. It isolates the "DSL file changed vs
-// runtime override changed" decision from the Run/resume plumbing so the
-// diagnostic label can be verified directly (the log text itself is not
-// asserted by capture).
-func TestClassifyDSLChange(t *testing.T) {
- const (
- dsl = "dslfp"
- ovf = "ovrfp"
- )
- cases := []struct {
- name string
- storedDsl string
- storedOvf string
- dslFP string
- ovfFP string
- wantReason string
- }{
- {"no change", dsl, ovf, dsl, ovf, ""},
- {"dsl file changed", dsl + "x", ovf, dsl, ovf, "DSL/template changed"},
- {"override changed", dsl, ovf + "x", dsl, ovf, "runtime override/parser-config changed"},
- {"both changed", dsl + "x", ovf + "x", dsl, ovf, "DSL/template changed"},
- // Caller only invokes classify when a checkpoint exists; a missing
- // fingerprint is treated as "changed" (safe: discard + fresh run).
- {"missing stored fingerprint", "", "", dsl, ovf, "DSL/template changed"},
- }
- for _, c := range cases {
- t.Run(c.name, func(t *testing.T) {
- got := classifyDSLChange(c.storedDsl, c.storedOvf, c.dslFP, c.ovfFP)
- if got != c.wantReason {
- t.Fatalf("classifyDSLChange = %q, want %q", got, c.wantReason)
- }
- })
- }
-}
-
-// errCheckpointStore fails every Get with getErr and records whether Set was
-// ever called. It lets us assert that guardDSLChange does NOT overwrite the
-// DSL/override fingerprints after a failed checkpoint lookup (which would mask
-// a real stale-checkpoint mismatch on a later resume).
-type errCheckpointStore struct {
- getErr error
- setCalled bool
-}
-
-func (s *errCheckpointStore) Get(_ context.Context, _ string) ([]byte, bool, error) {
- return nil, false, s.getErr
-}
-func (s *errCheckpointStore) Set(_ context.Context, _ string, _ []byte) error {
- s.setCalled = true
- return nil
-}
-func (s *errCheckpointStore) Delete(_ context.Context, _ string) error { return nil }
-
-// TestGuardDSLChange_CheckpointLookupErrorSkipsOverwrite verifies the
-// CodeRabbit finding: when store.Get(cpID) returns an error, guardDSLChange
-// bails out entirely and must NOT call Set on the fingerprint keys.
-func TestGuardDSLChange_CheckpointLookupErrorSkipsOverwrite(t *testing.T) {
- const taskID = "task-lookup-err"
- store := &errCheckpointStore{getErr: errors.New("redis temporarily unavailable")}
- p := &Pipeline{
- taskID: taskID,
- rawDSL: []byte(`{"dsl":{"components":{"begin":{"obj":{"component_name":"Begin","params":{}}}}}}`),
- }
- // tracker is nil-safe (only used on the delete branch, which is skipped).
- p.guardDSLChange(t.Context(), store, nil, taskID, map[string]any{"k": "v"})
- if store.setCalled {
- t.Fatal("guardDSLChange must not overwrite fingerprints after a failed checkpoint lookup")
- }
-}
-
-// TestPipelineRunResumableOverrideChanged verifies that editing ONLY the
-// runtime override_params (same DSL file) between the failed run and the
-// resume is detected as a change and forces a fresh run (the checkpoint is
-// discarded) — it must NOT resume against stale override state. This is the
-// "runtime override/parser-config changed" branch of guardDSLChange.
-func TestPipelineRunResumableOverrideChanged(t *testing.T) {
- mockA := &mockCanvasStage{output: map[string]any{"a": 1}}
- termStage := &oneShotErrStage{mockCanvasStage: mockCanvasStage{output: map[string]any{"b": 2}}}
-
- const (
- nameA = "p.OvfChangedA"
- nameB = "p.OvfChangedB"
- )
- runtime.MustRegister(nameA, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return mockA, nil },
- runtime.Metadata{Version: "1.0.0"})
- runtime.MustRegister(nameB, runtime.CategoryIngestion,
- func(_ string, _ map[string]any) (runtime.Component, error) { return termStage, nil },
- runtime.Metadata{Version: "1.0.0"})
-
- const taskID = "task-override-changed"
-
- store := newMemCheckpointStore()
- mr := miniredis.RunT(t)
- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
- t.Cleanup(func() { client.Close() })
- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)
-
- dsl := `{
- "dsl": {
- "components": {
- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
- "a": {"obj": {"component_name": "` + nameA + `", "params": {}}, "upstream": ["begin"], "downstream": ["b"]},
- "b": {"obj": {"component_name": "` + nameB + `", "params": {}}, "upstream": ["a"]}
- },
- "path": ["begin", "a", "b"],
- "graph": {"nodes": []}
- }
- }`
-
- // Run 1: override v1; terminal B errors (oneShotErrStage n=1), leaving a
- // checkpoint + interrupt. The DSL file fingerprint is recorded too.
- pipe1, err := NewPipelineFromDSL([]byte(dsl), taskID, WithCheckPointStore(store), WithRunTracker(tracker))
- if err != nil {
- t.Fatalf("NewPipelineFromDSL (run 1): %v", err)
- }
- if _, err := pipe1.Run(t.Context(), map[string]any{"name": "doc-ovf-changed"}, map[string]any{"k": "v1"}); err == nil {
- t.Fatal("Run 1: expected error from simulated crash, got nil")
- }
- if mockA.calls != 1 {
- t.Fatalf("Run 1: expected A to run once, got %d", mockA.calls)
- }
-
- // Run 2: SAME DSL file, but override changed to v2. The guard must detect
- // the override change, discard the stale checkpoint, and re-run from
- // scratch (A runs again).
- pipe2, err := NewPipelineFromDSL([]byte(dsl), taskID, WithCheckPointStore(store), WithRunTracker(tracker))
- if err != nil {
- t.Fatalf("NewPipelineFromDSL (run 2): %v", err)
- }
- if _, err := pipe2.Run(t.Context(), map[string]any{"name": "doc-ovf-changed"}, map[string]any{"k": "v2"}); err != nil {
- t.Fatalf("Run 2: expected fresh run to succeed after override edit, got error: %v", err)
- }
- if mockA.calls != 2 {
- t.Fatalf("Run 2: expected A to run again (fresh run on override change), got A.calls=%d", mockA.calls)
- }
- if termStage.calls != 1 {
- t.Fatalf("Run 2: expected B to delegate once, got %d", termStage.calls)
- }
- if store.deleteCount() < 1 {
- t.Fatalf("expected the stale checkpoint to be deleted on override change, got deleteCount=%d", store.deleteCount())
- }
-}
-
-// TestPipelineRun_RequireResumeRejectsWithoutStore verifies that with
-// WithRequireResume set and no checkpoint store resolvable (no
-// injected store, no global Redis in unit scope), Run must refuse to start
-// and return ErrResumeUnavailable — a clear, distinguishable signal — rather
-// than silently degrading to a non-resumable runPlain. The reject fires
-// before compile, so the DSL does not need a runnable graph.
-func TestPipelineRun_RequireResumeRejectsWithoutStore(t *testing.T) {
- pipe, err := NewPipelineFromDSL([]byte(`{
- "dsl": {
- "components": {
- "begin": {"obj": {"component_name": "Begin", "params": {}}, "downstream": ["a"]},
- "a": {"obj": {"component_name": "p.Docx", "params": {}}, "upstream": ["begin"]}
- },
- "path": ["begin", "a"],
- "graph": {"nodes": []}
- }
- }`), "task-req-resume", WithRequireResume())
- if err != nil {
- t.Fatalf("NewPipelineFromDSL: %v", err)
- }
- _, err = pipe.Run(t.Context(), map[string]any{"name": "doc"}, nil)
- if !errors.Is(err, ErrResumeUnavailable) {
- t.Fatalf("expected ErrResumeUnavailable, got %v", err)
- }
-}
-
// recordingSink captures OnComponentTotal / OnComponentProgress calls so tests
// can assert the pipeline forwards progress to the sink instead of writing
// the DAO layer directly.
@@ -1018,72 +358,6 @@ func TestPipelineRunForwardsProgressToSink(t *testing.T) {
}
}
-// =============================================================================
-// cleanupCheckpoint — direct unit test
-// =============================================================================
-
-func TestCleanupCheckpoint_DeletesStoreAndClearsTracker(t *testing.T) {
- mr := miniredis.RunT(t)
- client := redis.NewClient(&redis.Options{Addr: mr.Addr()})
- t.Cleanup(func() { client.Close() })
-
- store := newMemCheckpointStore()
- if err := store.Set(t.Context(), "cp-1", []byte("data")); err != nil {
- t.Fatalf("store.Set: %v", err)
- }
- // Fingerprint keys must share the checkpoint's lifecycle on cleanup.
- if err := store.Set(t.Context(), "cp-1"+dslKeySuffix, []byte("dslfp")); err != nil {
- t.Fatalf("store.Set dsl: %v", err)
- }
- if err := store.Set(t.Context(), "cp-1"+ovfKeySuffix, []byte("ovrfp")); err != nil {
- t.Fatalf("store.Set ovf: %v", err)
- }
- tracker := canvas.NewRunTrackerWithClient(client, time.Hour)
- if err := tracker.AttachInterrupt(t.Context(), "cp-1", "interrupt-1"); err != nil {
- t.Fatalf("AttachInterrupt: %v", err)
- }
-
- p := &Pipeline{}
- p.cleanupCheckpoint(t.Context(), store, tracker, "cp-1")
-
- // checkpoint + dsl fingerprint + ovf fingerprint = 3 deletes.
- if store.deleteCount() != 3 {
- t.Fatalf("expected 3 store deletes (checkpoint + 2 fingerprints), got %d", store.deleteCount())
- }
- for _, k := range []string{"cp-1", "cp-1" + dslKeySuffix, "cp-1" + ovfKeySuffix} {
- if _, found, _ := store.Get(t.Context(), k); found {
- t.Fatalf("expected key %q to be deleted", k)
- }
- }
- id, ok, err := tracker.GetInterruptID(t.Context(), "cp-1")
- if err != nil {
- t.Fatalf("GetInterruptID: %v", err)
- }
- if ok && id != "" {
- t.Fatalf("interrupt id should be cleared, got %q", id)
- }
-}
-
-func TestCleanupTaskStateDeletesCheckpointAndFingerprints(t *testing.T) {
- store := newMemCheckpointStore()
- for _, key := range []string{"task-1", "task-1:dsl", "task-1:ovf"} {
- if err := store.Set(t.Context(), key, []byte("state")); err != nil {
- t.Fatalf("seed %s: %v", key, err)
- }
- }
-
- if err := cleanupTaskState(t.Context(), store, nil, "task-1"); err != nil {
- t.Fatalf("cleanupTaskState: %v", err)
- }
- for _, key := range []string{"task-1", "task-1:dsl", "task-1:ovf"} {
- if _, ok, err := store.Get(t.Context(), key); err != nil {
- t.Fatalf("read %s: %v", key, err)
- } else if ok {
- t.Fatalf("checkpoint state %s still exists", key)
- }
- }
-}
-
// =============================================================================
// runPlain — tracker integration with miniredis
// =============================================================================
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index d48252818e..16f5350501 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -1195,7 +1195,7 @@ func (e *Ingestor) defaultRunDocumentTask(ctx context.Context, ingestionTask *en
if ingestionTask.PipelineLogID != nil {
pipelineLogID = *ingestionTask.PipelineLogID
}
- result, err := executor.WithRequireResume().WithProgressSink(newProgressSink(ctx, e.ingestionTaskSvc, pipelineLogID)).Execute(docTaskCtx.Ctx)
+ result, err := executor.WithProgressSink(newProgressSink(ctx, e.ingestionTaskSvc, pipelineLogID)).Execute(docTaskCtx.Ctx)
if err != nil {
return err
}
diff --git a/internal/ingestion/task/pipeline_executor.go b/internal/ingestion/task/pipeline_executor.go
index bac2bd0ad4..16c3af4e03 100644
--- a/internal/ingestion/task/pipeline_executor.go
+++ b/internal/ingestion/task/pipeline_executor.go
@@ -73,7 +73,6 @@ type PipelineExecutor struct {
loadDSLFunc func(ctx context.Context, canvasID string) (string, string, error)
runPipelineFunc func(ctx context.Context, dsl string) (map[string]any, string, error)
progressSink pipelinepkg.ProgressSink
- requireResume bool // when true, the pipeline run passes WithRequireResume
}
func validateTaskContext(taskCtx *TaskContext) error {
@@ -168,14 +167,6 @@ func (s *PipelineExecutor) WithProgressSink(sink pipelinepkg.ProgressSink) *Pipe
return s
}
-// WithRequireResume makes the pipeline refuse to start when no checkpoint
-// store is resolvable (Redis down or not configured). Production ingestion
-// sets this; tests skip it so they can exercise runPlain without Redis.
-func (s *PipelineExecutor) WithRequireResume() *PipelineExecutor {
- s.requireResume = true
- return s
-}
-
func (s *PipelineExecutor) KB() *entity.Knowledgebase { return &s.taskCtx.KB }
func (s *PipelineExecutor) Doc() *entity.Document { return &s.taskCtx.Doc }
func (s *PipelineExecutor) Tenant() *entity.Tenant { return &s.taskCtx.Tenant }
diff --git a/internal/service/document/document_parse.go b/internal/service/document/document_parse.go
index e67b18cd13..2a302978a6 100644
--- a/internal/service/document/document_parse.go
+++ b/internal/service/document/document_parse.go
@@ -58,10 +58,10 @@ func lockDocumentParse(docID string) func() {
}
}
-// purgeTaskStateForCleanup removes resumable checkpoint, tracker, and chunk
-// cache state before a task row is deleted. The bounded context makes Redis
-// outages and hung clients observable to the caller instead of allowing a
-// destructive task/document delete to proceed with stale resume state.
+// purgeTaskStateForCleanup removes the per-chunk cache state for a task before
+// its row is deleted. The bounded context makes Redis outages and hung clients
+// observable to the caller instead of allowing a destructive task/document
+// delete to proceed with stale cache state.
func (s *DocumentService) purgeTaskStateForCleanup(ctx context.Context, taskID string) error {
purgeTaskState := s.purgeTaskState
if purgeTaskState == nil {