Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(compile): bound job and shuffle retry attempts (#5989)
Jiajie - He/him/his5ff645a4ed4b
diff --git a/bot/vikingbot/compile/ops/common.py b/bot/vikingbot/compile/ops/common.pyindex 481d43089..360ec2363 100644--- a/bot/vikingbot/compile/ops/common.py+++ b/bot/vikingbot/compile/ops/common.py@@ -7,7 +7,6 @@ to pipeline_io. Record transforms are shared by Map and intermediate Reduce stagfrom __future__ import annotationsimport json-from collections import Counterfrom dataclasses import asdictfrom typing import TYPE_CHECKING@@ -15,7 +14,7 @@ from openviking.core.namespace import relative_uri_pathfrom openviking.utils.model_retry import ERROR_CLASS_INPUT_TOO_LARGE, classify_api_errorfrom vikingbot.compile import file_opsfrom vikingbot.compile.hashing import content_hash, digest-from vikingbot.compile.pipeline_io import retry_allowed+from vikingbot.compile.pipeline_io import MAX_JOB_ATTEMPTSfrom vikingbot.compile.renderer import validate_relative_file_pathfrom vikingbot.compile.results import (CombineResponse,@@ -337,18 +336,18 @@ async def transform(async def job(runtime: Pipeline, name, inputs, work):- """Persist one job's outcome; failures never advance source/record completion."""+ """Persist one job's outcome with bounded retries; failures never advance input completion."""entry = {"status": "running", "inputs": [r.record_id for r in inputs]}await runtime.files.put(f"jobs/{name}", entry)- retries = Counter()try:- while True:+ for attempt in range(MAX_JOB_ATTEMPTS):try:output = await work()breakexcept OSError as exc:- if classify_api_error(exc) == ERROR_CLASS_INPUT_TOO_LARGE or not retry_allowed(- retries, str(exc)+ if (+ attempt == MAX_JOB_ATTEMPTS - 1+ or classify_api_error(exc) == ERROR_CLASS_INPUT_TOO_LARGE):raiseruntime.metrics["job_retries"] += 1diff --git a/bot/vikingbot/compile/ops/shuffle.py b/bot/vikingbot/compile/ops/shuffle.pyindex c01212a05..31f196b6d 100644--- a/bot/vikingbot/compile/ops/shuffle.py+++ b/bot/vikingbot/compile/ops/shuffle.py@@ -7,13 +7,12 @@ import heapqimport jsonimport mathfrom array import array-from collections import Counterfrom collections.abc import Sequencefrom threading import Eventfrom openviking.core.namespace import relative_uri_pathfrom vikingbot.compile.hashing import digest-from vikingbot.compile.pipeline_io import bounded_jobs, retry_allowed+from vikingbot.compile.pipeline_io import MAX_JOB_ATTEMPTS, bounded_jobsfrom vikingbot.compile.plan import Routingfrom vikingbot.compile.results import Group, Record, RouteBatchResponse, RouteDecision@@ -249,7 +248,7 @@ class Shuffle:"""Assign every record to one work set, allowing cross-path and cross-scope links.Independent batches compare global candidates, including later records.- Only failed primary records retry; successful decisions are persisted immediately.+ Failed primary records retry within a finite budget; successful decisions persist immediately.Unconfirmed candidates never become links. Recall failuresare recorded without fabricating empty history; cancellation propagates."""@@ -297,7 +296,6 @@ class Shuffle:}accepted, errors = {}, {}- retries = {record.record_id: Counter() for record in records}system = ("Routing rules:\n"+ (rule.instructions if isinstance(rule, Routing) else rule)@@ -429,7 +427,7 @@ class Shuffle:state = ("completed"if key in accepted- else ("pending" if retry_allowed(retries[key], errors[key]) else "failed")+ else ("pending" if attempt < MAX_JOB_ATTEMPTS else "failed"))entry = {"status": state, "inputs": [key], "attempt": attempt}if key in accepted:diff --git a/bot/vikingbot/compile/pipeline_io.py b/bot/vikingbot/compile/pipeline_io.pyindex 4a5c37443..3aef9ea87 100644--- a/bot/vikingbot/compile/pipeline_io.py+++ b/bot/vikingbot/compile/pipeline_io.py@@ -32,6 +32,8 @@ ROOT = f"{COMPILE_STAGING_ROOT}/pipeline"PROCESSING_VERSION = "compile-pipeline-36"# Output-token fallback when the configured VLM provides no value.DEFAULT_MAX_TOKENS = 32_000+# Total attempts per operator job or routing record, including the initial execution.+MAX_JOB_ATTEMPTS = 4def parse_file_result(text: str) -> Any:diff --git a/bot/vikingbot/config/loader.py b/bot/vikingbot/config/loader.pyindex 418341e70..c485c99ee 100644--- a/bot/vikingbot/config/loader.py+++ b/bot/vikingbot/config/loader.py@@ -218,7 +218,7 @@ def _merge_vlm_model_config(agents = bot_data.get("agents", {})if vlm_data and "timeout" not in agents:- agents["timeout"] = vlm_data["timeout"] if "timeout" in vlm_data else 60.0+ agents["timeout"] = vlm_data["timeout"] if "timeout" in vlm_data else 600.0if not inherit_model:return