Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(compile): bound job and shuffle retry attempts (#5989)

Jiajie - He/him/his5ff645a4ed4b
diff --git a/bot/vikingbot/compile/ops/common.py b/bot/vikingbot/compile/ops/common.py
index 481d43089..360ec2363 100644
--- a/bot/vikingbot/compile/ops/common.py
+++ b/bot/vikingbot/compile/ops/common.py
@@ -7,7 +7,6 @@ to pipeline_io. Record transforms are shared by Map and intermediate Reduce stag
from __future__ import annotations
import json
-from collections import Counter
from dataclasses import asdict
from typing import TYPE_CHECKING
@@ -15,7 +14,7 @@ from openviking.core.namespace import relative_uri_path
from openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, classify_api_error
from vikingbot.compile import file_ops
from vikingbot.compile.hashing import content_hash, digest
-from vikingbot.compile.pipeline_io import retry_allowed
+from vikingbot.compile.pipeline_io import MAX_JOB_ATTEMPTS
from vikingbot.compile.renderer import validate_relative_file_path
from vikingbot.compile.results import (
CombineResponse,
@@ -337,18 +336,18 @@ async def transform(
async def job(runtime: Pipeline, name, inputs, work):
- """Persist one job's outcome; failures never advance source/record completion."""
+ """Persist one job's outcome with bounded retries; failures never advance input completion."""
entry = {"status": "running", "inputs": [r.record_id for r in inputs]}
await runtime.files.put(f"jobs/{name}", entry)
- retries = Counter()
try:
- while True:
+ for attempt in range(MAX_JOB_ATTEMPTS):
try:
output = await work()
break
except OSError as exc:
- if classify_api_error(exc) == ERROR_CLASS_INPUT_TOO_LARGE or not retry_allowed(
- retries, str(exc)
+ if (
+ attempt == MAX_JOB_ATTEMPTS - 1
+ or classify_api_error(exc) == ERROR_CLASS_INPUT_TOO_LARGE
):
raise
runtime.metrics["job_retries"] += 1
diff --git a/bot/vikingbot/compile/ops/shuffle.py b/bot/vikingbot/compile/ops/shuffle.py
index c01212a05..31f196b6d 100644
--- a/bot/vikingbot/compile/ops/shuffle.py
+++ b/bot/vikingbot/compile/ops/shuffle.py
@@ -7,13 +7,12 @@ import heapq
import json
import math
from array import array
-from collections import Counter
from collections.abc import Sequence
from threading import Event
from openviking.core.namespace import relative_uri_path
from vikingbot.compile.hashing import digest
-from vikingbot.compile.pipeline_io import bounded_jobs, retry_allowed
+from vikingbot.compile.pipeline_io import MAX_JOB_ATTEMPTS, bounded_jobs
from vikingbot.compile.plan import Routing
from vikingbot.compile.results import Group, Record, RouteBatchResponse, RouteDecision
@@ -249,7 +248,7 @@ class Shuffle:
"""Assign every record to one work set, allowing cross-path and cross-scope links.
Independent batches compare global candidates, including later records.
- Only failed primary records retry; successful decisions are persisted immediately.
+ Failed primary records retry within a finite budget; successful decisions persist immediately.
Unconfirmed candidates never become links. Recall failures
are recorded without fabricating empty history; cancellation propagates.
"""
@@ -297,7 +296,6 @@ class Shuffle:
}
accepted, errors = {}, {}
- retries = {record.record_id: Counter() for record in records}
system = (
"Routing rules:\n"
+ (rule.instructions if isinstance(rule, Routing) else rule)
@@ -429,7 +427,7 @@ class Shuffle:
state = (
"completed"
if key in accepted
- else ("pending" if retry_allowed(retries[key], errors[key]) else "failed")
+ else ("pending" if attempt < MAX_JOB_ATTEMPTS else "failed")
)
entry = {"status": state, "inputs": [key], "attempt": attempt}
if key in accepted:
diff --git a/bot/vikingbot/compile/pipeline_io.py b/bot/vikingbot/compile/pipeline_io.py
index 4a5c37443..3aef9ea87 100644
--- a/bot/vikingbot/compile/pipeline_io.py
+++ b/bot/vikingbot/compile/pipeline_io.py
@@ -32,6 +32,8 @@ ROOT = f"{COMPILE_STAGING_ROOT}/pipeline"
PROCESSING_VERSION = "compile-pipeline-36"
# Output-token fallback when the configured VLM provides no value.
DEFAULT_MAX_TOKENS = 32_000
+# Total attempts per operator job or routing record, including the initial execution.
+MAX_JOB_ATTEMPTS = 4
def parse_file_result(text: str) -> Any:
diff --git a/bot/vikingbot/config/loader.py b/bot/vikingbot/config/loader.py
index 418341e70..c485c99ee 100644
--- a/bot/vikingbot/config/loader.py
+++ b/bot/vikingbot/config/loader.py
@@ -218,7 +218,7 @@ def _merge_vlm_model_config(
agents = bot_data.get("agents", {})
if vlm_data and "timeout" not in agents:
- agents["timeout"] = vlm_data["timeout"] if "timeout" in vlm_data else 60.0
+ agents["timeout"] = vlm_data["timeout"] if "timeout" in vlm_data else 600.0
if not inherit_model:
return