Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: skip unchanged Git watch refreshes with SHA checkpoints (#6697)

chenxiaobin-monkey25b83a603864
diff --git a/openviking/parse/accessors/git_accessor.py b/openviking/parse/accessors/git_accessor.py
index cf46230ca..a449a4bb2 100644
--- a/openviking/parse/accessors/git_accessor.py
+++ b/openviking/parse/accessors/git_accessor.py
@@ -11,6 +11,7 @@ import asyncio
import contextlib
import os
import shutil
+import signal
import stat
import tempfile
import urllib.request
@@ -61,6 +62,10 @@ class GitAccessor(DataAccessor):
"""
PRIORITY = 80 # Higher than generic HTTP, lower than specific services
+ # Upper bound for reaping the git process tree after cancellation. The
+ # tree is SIGKILLed as a group, so this only guards pathological reap
+ # delays and must stay short to avoid pinning the caller's slot.
+ CANCEL_REAP_TIMEOUT = 1.0
@property
def priority(self) -> int:
@@ -115,8 +120,14 @@ class GitAccessor(DataAccessor):
temp_local_dir = None
branch = kwargs.get("branch") or kwargs.get("ref")
commit = kwargs.get("commit")
+ if commit:
+ branch = None
auth_config = parse_git_http_auth_config(kwargs.get("auth_config"), source_str)
git_env = None
+ used_git_clone = False
+ # Exact immutable revision fetched. Branch tracking resolves this from
+ # the remote/local HEAD; commit-only imports use the supplied commit.
+ resolved_sha: Optional[str] = None
try:
# Create local temp directory (non-blocking)
@@ -140,11 +151,13 @@ class GitAccessor(DataAccessor):
commit=commit,
env=git_env,
)
+ used_git_clone = True
elif source_str.startswith(("http://", "https://", "git://", "ssh://")):
repo_url, branch, commit = self._parse_repo_source(source_str, **kwargs)
+ github_token = None
if auth_config is not None:
git_env = build_git_http_auth_env(auth_config, repo_url)
- if self._is_github_url(repo_url) and auth_config is None:
+ elif self._is_github_url(repo_url):
github_token = kwargs.get("github_token") or os.environ.get("GITHUB_TOKEN")
if github_token and is_git_https_url(repo_url):
github_auth = parse_git_http_auth_config(
@@ -153,11 +166,24 @@ class GitAccessor(DataAccessor):
)
assert github_auth is not None
git_env = build_git_http_auth_env(github_auth, repo_url)
+ if commit:
+ resolved_sha = commit
+ else:
+ # Watch refresh / add_resource first round pass the revision
+ # they already resolved (scheduler probe or preflight) here.
+ # Without a hint the ref is fetched by its moving branch name
+ # and no exact-revision checkpoint is recorded; this accessor
+ # never issues its own ls-remote.
+ resolved_sha = kwargs.get("git_head_sha_hint") or None
+ # Fetch an immutable per-SHA archive when resolved; fall back to
+ # the moving branch/commit name only when no revision was supplied.
+ download_ref = resolved_sha or commit or branch
+ if self._is_github_url(repo_url) and auth_config is None:
# Try GitHub ZIP API first, fall back to git clone
try:
local_dir, repo_name = await self._github_zip_download(
repo_url,
- branch or commit,
+ download_ref,
temp_local_dir,
github_token=github_token,
)
@@ -182,11 +208,12 @@ class GitAccessor(DataAccessor):
commit=commit,
env=git_env,
)
+ used_git_clone = True
elif self._can_use_gitlab_zip(repo_url) and auth_config is None:
# Try GitLab ZIP API first, fall back to git clone
try:
local_dir, repo_name = await self._gitlab_zip_download(
- repo_url, branch or commit, temp_local_dir
+ repo_url, download_ref, temp_local_dir
)
except Exception as zip_exc:
logger.warning(
@@ -209,6 +236,7 @@ class GitAccessor(DataAccessor):
commit=commit,
env=git_env,
)
+ used_git_clone = True
else:
# Other configured code-hosting URLs: use git clone
repo_name = await self._git_clone(
@@ -218,9 +246,20 @@ class GitAccessor(DataAccessor):
commit=commit,
env=git_env,
)
+ used_git_clone = True
else:
raise ValueError(f"Unsupported source for GitAccessor: {source}")
+ if commit:
+ # A commit-only import is already pinned to an immutable revision.
+ # Keep the supplied value as the checkpoint without another probe.
+ resolved_sha = commit
+ elif used_git_clone:
+ # The clone fallback preserves the historical branch/commit
+ # behavior. Record the revision that was actually checked out,
+ # rather than the SHA probed before cloning.
+ resolved_sha = await self._resolve_local_head(temp_local_dir, git_env)
+
# Build metadata
# repo_name is in "org/repo" format (e.g. "volcengine/OpenViking")
# This is extracted via parse_code_hosting_url() from the original URL
@@ -229,6 +268,10 @@ class GitAccessor(DataAccessor):
meta["repo_ref"] = branch
if commit:
meta["repo_commit"] = commit
+ if resolved_sha:
+ # Exact revision fetched; the watch layer persists this as its
+ # source checkpoint.
+ meta["resolved_head_sha"] = resolved_sha
return LocalResource(
path=local_dir,
@@ -260,19 +303,72 @@ class GitAccessor(DataAccessor):
if parsed_source is None:
if source.startswith(("git@", "http://", "https://", "git://", "ssh://")):
raise ValueError(f"Unsupported Git repository URL: {source}")
- return source, branch, commit
-
- if commit is None or branch is None:
- commit = parsed_source.commit or commit
- # Preserve the existing precedence: a commit encoded in the URL
- # overrides branch selection, while an explicit branch overrides a
- # branch encoded in the URL.
if commit:
branch = None
- else:
- branch = branch or parsed_source.branch
+ return source, branch, commit
+
+ commit = commit or parsed_source.commit
+ branch = None if commit else branch or parsed_source.branch
+ return parsed_source.clone_url, branch, commit
+
+ @staticmethod
+ def _resolve_probe_target(
+ source: str, **kwargs
+ ) -> Optional[Tuple[str, Optional[str], Optional[str]]]:
+ """Resolve (clone_url, branch, commit) for a watch probe.
+
+ Mirrors ``_parse_repo_source`` but returns None for non-git sources
+ instead of raising, so callers can fall back to a full refresh.
+ """
+ branch = kwargs.get("branch") or kwargs.get("ref")
+ commit = kwargs.get("commit")
+ parsed_source = parse_git_repo_url(source)
+ if parsed_source is None:
+ if source.startswith(("git@", "http://", "https://", "git://", "ssh://")):
+ if commit:
+ branch = None
+ return source, branch, commit
+ return None
+ commit = commit or parsed_source.commit
+ branch = None if commit else branch or parsed_source.branch
return parsed_source.clone_url, branch, commit
+ @staticmethod
+ def _parse_ls_remote_ref(stdout: str, ref: str) -> Optional[str]:
+ """Extract the SHA advertised for *ref* from ``git ls-remote`` output."""
+ for line in stdout.splitlines():
+ parts = line.split("\t", 1)
+ if len(parts) == 2 and parts[1].strip() == ref:
+ return parts[0].strip() or None
+ return None
+
+ async def probe_git_remote_head(
+ self, source: Union[str, Path], **kwargs
+ ) -> Optional[Tuple[str, str]]:
+ """Resolve the watched git revision without downloading repository content.
+
+ Branch watches return the remotely advertised ``(ref, head_sha)``.
+ Commit-only watches return an empty ref and the supplied commit locally.
+ Raises on branch probe failures so callers can apply a fail-open policy.
+ """
+ source_str = str(source)
+ resolved = self._resolve_probe_target(source_str, **kwargs)
+ if resolved is None:
+ return None
+ repo_url, branch, commit = resolved
+ if commit:
+ # The supplied commit is already an immutable revision. Return it
+ # directly so callers can compare checkpoints without ls-remote.
+ return "", commit
+
+ auth_config = parse_git_http_auth_config(kwargs.get("auth_config"), source_str)
+ env = build_git_http_auth_env(auth_config, repo_url) if auth_config is not None else None
+ env = overlay_parser_subprocess_env(
+ ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, repo_url),
+ env,
+ )
+ return await self._git_ls_remote_head(repo_url, branch, env)
+
def _normalize_repo_url(self, url: str) -> str:
"""Normalize a validated repository URL to a cloneable base form."""
parsed_source = parse_git_repo_url(url)
@@ -325,14 +421,17 @@ class GitAccessor(DataAccessor):
}
if env is not None:
process_kwargs["env"] = env
+ if os.name == "posix":
+ # Detach git into a fresh session/process group so that remote
+ # helpers (http/ssh) and other descendants can be signalled
+ # together on timeout/cancellation instead of being orphaned
+ # while they still hold our stdout/stderr pipes.
+ process_kwargs["start_new_session"] = True
proc = await asyncio.create_subprocess_exec(*args, **process_kwargs)
try:
stdout, stderr = await proc.communicate()
except BaseException:
- with contextlib.suppress(ProcessLookupError):
- proc.kill()
- with contextlib.suppress(Exception):
- await asyncio.shield(proc.wait())
+ await self._terminate_git_process(proc)
raise
if proc.returncode != 0:
raise_git_auth_error(stderr)
@@ -352,6 +451,36 @@ class GitAccessor(DataAccessor):
raise RuntimeError(user_msg)
return stdout.decode().strip()
+ async def _terminate_git_process(self, proc: asyncio.subprocess.Process) -> None:
+ """Best-effort termination of the whole git process tree.
+
+ Git spawns remote helpers (http/ssh) which inherit the stdout/stderr
+ pipes. Killing only the direct git child leaves those descendants
+ alive and holding the pipes open, so ``communicate()``/``wait()`` may
+ block long after cancellation. On POSIX git runs in its own process
+ group (see ``start_new_session``), so signal the entire group.
+ """
+ if os.name == "posix":
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ else:
+ # Non-POSIX platforms have no process-group signal primitive;
+ # fall back to the direct child.
+ with contextlib.suppress(ProcessLookupError):
+ proc.kill()
+
+ # Reaping itself must be bounded: shield protects the reap from a
+ # second cancellation, while wait_for guarantees cleanup cannot pin
+ # the caller's execution slot indefinitely. CancelledError derives
+ # from BaseException and is deliberately not suppressed.
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(
+ asyncio.shield(proc.wait()),
+ timeout=self.CANCEL_REAP_TIMEOUT,
+ )
+
async def _has_commit(
self,
repo_dir: str,
@@ -396,6 +525,8 @@ class GitAccessor(DataAccessor):
"""Clone a git repository into target_dir; return the repo name."""
name = self._get_repo_name(url)
logger.info(f"[GitAccessor] Cloning {url} to {target_dir}...")
+ if commit:
+ branch = None
network_env = overlay_parser_subprocess_env(
ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, url),
env,
@@ -466,6 +597,51 @@ class GitAccessor(DataAccessor):
return name
+ async def _resolve_local_head(
+ self,
+ repo_dir: str,
+ env: Optional[dict[str, str]] = None,
+ ) -> Optional[str]:
+ """Return the revision actually checked out by git clone."""
+ try:
+ head_sha = await self._run_git(
+ ["git", "-C", repo_dir, "rev-parse", "HEAD"],
+ env=env,
+ )
+ except Exception as exc:
+ logger.warning(
+ f"[GitAccessor] failed to resolve cloned repository HEAD: {exc}; "
+ "continuing without an exact-revision checkpoint"
+ )
+ return None
+ return head_sha.strip() or None
+
+ async def _git_ls_remote_head(
+ self,
+ url: str,
+ branch: Optional[str],
+ env: Optional[dict[str, str]] = None,
+ ) -> Tuple[str, str]:
+ """Probe a remote for the tip SHA of *branch* (or HEAD if branch is None).
+
+ Returns ``(ref, sha)`` where ref is the fully qualified ref advertised by
+ the remote. Uses ``ls-remote`` only (no content download) so the cost is
+ a single network round-trip.
+ """
+ if branch:
+ ref = branch if branch.startswith("refs/") else f"refs/heads/{branch}"
+ args = ["git", "ls-remote", "--heads", url, ref]
+ else:
+ # Track the remote's default branch via HEAD, which every server
+ # advertises; no --symref dependency.
+ ref = "HEAD"
+ args = ["git", "ls-remote", url, "HEAD"]
+ stdout = await self._run_git(args, env=env)
+ sha = self._parse_ls_remote_ref(stdout, ref)
+ if sha is None:
+ raise RuntimeError(f"git ls-remote did not advertise {ref} for {url}")
+ return ref, sha
+
async def _github_zip_download(
self,
repo_url: str,
diff --git a/openviking/parse/parsers/code/code.py b/openviking/parse/parsers/code/code.py
index be4c7f307..39d4dffd2 100644
--- a/openviking/parse/parsers/code/code.py
+++ b/openviking/parse/parsers/code/code.py
@@ -232,6 +232,9 @@ class CodeRepositoryParser(BaseParser):
result.meta["repo_ref"] = branch
if commit:
result.meta["repo_commit"] = commit
+ resolved_head_sha = source_meta.get("resolved_head_sha")
+ if resolved_head_sha:
+ result.meta["resolved_head_sha"] = resolved_head_sha
return result
diff --git a/openviking/resource/watch_manager.py b/openviking/resource/watch_manager.py
index 42f057348..c801b0bec 100644
--- a/openviking/resource/watch_manager.py
+++ b/openviking/resource/watch_manager.py
@@ -85,6 +85,13 @@ class WatchTask(BaseModel):
connector_states: Optional[Dict[str, Any]] = Field(
default=None, description="Private external Connector stream states"
)
+ source_checkpoint: Optional[Dict[str, Any]] = Field(
+ default=None,
+ description=(
+ "Private source version checkpoint used to skip unchanged refreshes, "
+ "e.g. {'kind': 'git', 'head_sha': '<sha>', 'instruction': '<instruction>'}"
+ ),
+ )
created_at: datetime = Field(default_factory=datetime.now, description="Task creation time")
last_execution_time: Optional[datetime] = Field(None, description="Last execution time")
last_task_id: Optional[str] = Field(None, description="Latest ingestion task identifier")
@@ -139,6 +146,8 @@ class WatchTask(BaseModel):
data["auth_state"] = self.auth_state
if self.connector_states is not None:
data["connector_states"] = self.connector_states
+ if self.source_checkpoint is not None:
+ data["source_checkpoint"] = self.source_checkpoint
return data
@classmethod
@@ -159,6 +168,10 @@ class WatchTask(BaseModel):
data.get("connector_states"), dict
):
data["connector_states"] = None
+ if data.get("source_checkpoint") is not None and not isinstance(
+ data.get("source_checkpoint"), dict
+ ):
+ data["source_checkpoint"] = None
return cls(**data)
def calculate_next_execution_time(self) -> datetime:
@@ -537,6 +550,7 @@ class WatchManager:
auth_state: Any = _UNSET,
connector_states: Any = _UNSET,
is_active: Optional[bool] = None,
+ invalidate_source_checkpoint: bool = False,
) -> WatchTask:
"""Update a watch task while its current and requested target URIs are stable."""
while True:
@@ -580,6 +594,7 @@ class WatchManager:
auth_state=auth_state,
connector_states=connector_states,
is_active=is_active,
+ invalidate_source_checkpoint=invalidate_source_checkpoint,
)
async def _update_task_unlocked(
@@ -603,6 +618,7 @@ class WatchManager:
auth_state: Any,
connector_states: Any,
is_active: Optional[bool],
+ invalidate_source_checkpoint: bool,
) -> WatchTask:
task_id = task.task_id
if self._check_uri_conflict(
@@ -657,6 +673,8 @@ class WatchManager:
task.connector_states = connector_states
if is_active is not None:
task.is_active = is_active
+ if invalidate_source_checkpoint:
+ task.source_checkpoint = None
if watch_interval is not None:
if task.is_active and task.watch_interval > 0:
@@ -709,6 +727,21 @@ class WatchManager:
task.connector_states = connector_states
await self._save_tasks()
+ async def update_source_checkpoint(
+ self,
+ task_id: str,
+ source_checkpoint: Optional[Dict[str, Any]],
+ ) -> None:
+ """Persist the source version checkpoint recorded after a successful refresh."""
+ if source_checkpoint is not None and not isinstance(source_checkpoint, dict):
+ raise ValueError("source_checkpoint must be an object")
+ async with self._lock:
+ task = self._tasks.get(task_id)
+ if not task:
+ return
+ task.source_checkpoint = source_checkpoint
+ await self._save_tasks()
+
async def record_execution(
self,
task_id: str,
diff --git a/openviking/resource/watch_scheduler.py b/openviking/resource/watch_scheduler.py
index 74cabf1d5..584b24cf5 100644
--- a/openviking/resource/watch_scheduler.py
+++ b/openviking/resource/watch_scheduler.py
@@ -50,6 +50,7 @@ class WatchScheduler:
DEFAULT_CHECK_INTERVAL = 60.0
DEFAULT_TASK_TIMEOUT = 3 * 60 * 60
+ DEFAULT_GIT_PROBE_TIMEOUT = 30.0
def __init__(
self,
@@ -329,6 +330,9 @@ class WatchScheduler:
execution_status = None
execution_error = None
execution_code = None
+ probed_checkpoint: Optional[Dict[str, Any]] = None
+ refreshed_checkpoint: Optional[Dict[str, Any]] = None
+ skipped_unchanged = False
try:
auth_state = getattr(task, "auth_state", None)
@@ -413,7 +417,40 @@ class WatchScheduler:
processor_kwargs["add_type"] = add_type
processor_kwargs["args"] = connector_args
- if not should_deactivate:
+ # Cheap source-version gate for native git watches: skip the full
+ # clone/parse refresh while the watched branch tip is unchanged.
+ # Probe failures fail open to a normal refresh.
+ if not should_deactivate and not connector_watch:
+ probed_checkpoint = await self._probe_git_checkpoint(
+ task.path,
+ processor_kwargs,
+ ctx=ctx,
+ instruction=task.instruction,
+ )
+ if probed_checkpoint is not None:
+ # Reuse the probed revision as an immutable download
+ # target so GitAccessor.access() skips its own
+ # ls-remote probe on the actual refresh.
+ processor_kwargs["git_head_sha_hint"] = probed_checkpoint[
+ "head_sha"
+ ]
+ stored_checkpoint = getattr(task, "source_checkpoint", None) or {}
+ if (
+ probed_checkpoint is not None
+ and stored_checkpoint == probed_checkpoint
+ and stored_checkpoint.get("kind") == "git"
+ ):
+ skipped_unchanged = True
+ execution_status = "completed"
+ # No new ingestion runs: keep the last real task id so
+ # record_execution does not null out the public field.
+ execution_task_id = task.last_task_id
+ logger.info(
+ f"[WatchScheduler] Task {task.task_id} source unchanged at "
+ f"{probed_checkpoint.get('head_sha')}, skipping refresh"
+ )
+
+ if not should_deactivate and not skipped_unchanged:
refresh_kwargs = {}
if connector_watch:
refresh_kwargs["connector_states"] = getattr(task, "connector_states", None)
@@ -437,10 +474,19 @@ class WatchScheduler:
**refresh_kwargs,
**processor_kwargs,
)
-
execution_code = result.get("code")
execution_task_id = result.get("task_id")
result_status = str(result.get("status") or "").lower()
+ # Synchronous deployments (no queue manager) return the
+ # terminal result directly: extract its revision checkpoint
+ # here. Async submissions carry no meta yet and get
+ # overridden by the task_tracker.wait result below.
+ refreshed_checkpoint = (
+ self._resource_service._initial_source_checkpoint(
+ result.get("meta"),
+ instruction=task.instruction,
+ )
+ )
if execution_task_id and result_status not in {
"completed",
"failed",
@@ -477,8 +523,13 @@ class WatchScheduler:
else:
result_status = ingestion_task.status.value
execution_error = ingestion_task.error
- execution_code = (getattr(ingestion_task, "result", None) or {}).get(
- "code"
+ ingestion_result = getattr(ingestion_task, "result", None) or {}
+ execution_code = ingestion_result.get("code")
+ refreshed_checkpoint = (
+ self._resource_service._initial_source_checkpoint(
+ ingestion_result.get("meta"),
+ instruction=task.instruction,
+ )
)
if result_status in {"failed", "error"}:
@@ -503,6 +554,27 @@ class WatchScheduler:
task.task_id,
result["connector_states"],
)
+ if refreshed_checkpoint is not None:
+ # Persist the revision actually imported. ZIP downloads
+ # report their immutable target SHA; clone fallbacks
+ # report the checked-out local HEAD.
+ #
+ # Design intent: the checkpoint tracks the *source
+ # revision* only. A non-None checkpoint means the
+ # code was fetched and parsed successfully (the
+ # accessor only emits resolved_head_sha after a
+ # successful access()). Downstream semantic/embedding
+ # queue failures are post-processing concerns with
+ # their own retry paths; they must NOT block the
+ # source checkpoint, otherwise an unchanged HEAD
+ # would re-trigger a full clone+parse on every tick
+ # just to retry indexing. Queue health is surfaced
+ # via queue_status and does not gate source-version
+ # skip logic.
+ await self._watch_manager.update_source_checkpoint(
+ task.task_id,
+ refreshed_checkpoint,
+ )
logger.info(
f"[WatchScheduler] Task {task.task_id} executed successfully, "
f"result: {result.get('root_uri', 'N/A')}"
@@ -590,6 +662,51 @@ class WatchScheduler:
with self._lock:
self._executing_tasks.discard(task_id)
+ async def _probe_git_checkpoint(
+ self,
+ path: str,
+ processor_kwargs: Dict[str, Any],
+ *,
+ ctx: RequestContext,
+ instruction: str,
+ ) -> Optional[Dict[str, Any]]:
+ """Return the watched git revision as a source checkpoint.
+
+ Commit-only sources resolve locally from the supplied commit. Returns
+ None for non-git sources and branch probe failures (fail open).
+ """
+ from openviking.parse.accessors.git_accessor import GitAccessor
+ from openviking.utils.code_hosting_utils import is_git_repo_url
+
+ if not path or not is_git_repo_url(path):
+ return None
+ try:
+ probe_kwargs = dict(processor_kwargs)
+ if probe_kwargs.get("auth_config") is None:
+ resource_processor = getattr(self._resource_service, "_resource_processor", None)
+ if resource_processor is not None:
+ github_token = await resource_processor.github_token_for(path, ctx)
+ if github_token:
+ probe_kwargs["auth_config"] = {"token": github_token}
+ probed = await asyncio.wait_for(
+ GitAccessor().probe_git_remote_head(path, **probe_kwargs),
+ timeout=self.DEFAULT_GIT_PROBE_TIMEOUT,
+ )
+ except Exception as e:
+ logger.warning(
+ f"[WatchScheduler] git head probe failed for {path}: {e}; "
+ "falling back to full refresh"
+ )
+ return None
+ if probed is None:
+ return None
+ _, head_sha = probed
+ return {
+ "kind": "git",
+ "head_sha": head_sha,
+ "instruction": instruction,
+ }
+
async def _prepare_feishu_auth_state(
self,
task,
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index 1e4213a63..cf2ddb522 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -187,6 +187,7 @@ class _ResourceSourceInfo:
source_name: Optional[str] = None
source_path: Optional[str] = None
source_format: Optional[str] = None
+ head_sha: Optional[str] = None
@dataclass
@@ -310,6 +311,7 @@ class ResourceService:
status: str,
execution_task_id: Optional[str] = None,
error: Optional[str] = None,
+ source_checkpoint: Optional[Dict[str, Any]] = None,
) -> None:
"""Record a pre-created Watch's first round and release its scheduler hold."""
sanitized_error = (
@@ -326,6 +328,11 @@ class ResourceService:
execution_task_id=execution_task_id,
error=error,
)
+ if status == "completed" and source_checkpoint is not None:
+ await watch_manager.update_source_checkpoint(
+ task_id,
+ source_checkpoint,
+ )
logger.info(
"[ResourceService] Watch first round recorded: watch_task_id=%s status=%s "
"execution_task_id=%s%s",
@@ -351,6 +358,7 @@ class ResourceService:
"understanding_response_id",
"understanding_file_id",
"temp_file_id",
+ "git_head_sha_hint",
}:
continue
try:
@@ -405,6 +413,30 @@ class ResourceService:
if (to or "").strip() and (parent or "").strip():
raise InvalidArgumentError("Cannot specify both 'to' and 'parent' at the same time.")
+ @staticmethod
+ def _initial_source_checkpoint(
+ meta: Any,
+ *,
+ instruction: str = "",
+ ) -> Optional[Dict[str, Any]]:
+ """Build the watch source checkpoint from the imported revision meta.
+
+ The accessor records ``resolved_head_sha`` for the exact imported
+ revision: a resolved branch tip or the supplied commit-only pin. The
+ tracked branch or commit selection itself lives in processor_kwargs.
+ The instruction is included so changing it forces one refresh even
+ when the source revision is unchanged. Failed branch probes leave the
+ checkpoint unset (fail open).
+ """
+ head_sha = meta.get("resolved_head_sha") if isinstance(meta, dict) else None
+ if head_sha:
+ return {
+ "kind": "git",
+ "head_sha": head_sha,
+ "instruction": instruction,
+ }
+ return None
+
async def _manage_watch_if_needed(
self,
*,
@@ -869,6 +901,11 @@ class ResourceService:
internal_kwargs["create_parent"] = True
if msg.source_name is not None:
internal_kwargs["source_name"] = msg.source_name
+ if msg.git_head_sha_hint:
+ # One-shot SHA resolved by the add_resource preflight; lets
+ # the accessor download an immutable revision without a second
+ # ls-remote. Transient: never persisted into watch kwargs.
+ internal_kwargs["git_head_sha_hint"] = msg.git_head_sha_hint
if feishu_prepared:
from openviking.service.task_tracker import get_task_tracker
@@ -1140,6 +1177,8 @@ class ResourceService:
source_info = await self._preflight_git_source(
path,
auth_config=preflight_git_auth,
+ branch=processor_kwargs.get("branch") or processor_kwargs.get("ref"),
+ commit=processor_kwargs.get("commit"),
)
source_name = source_name or source_info.source_name
source_info.source_name = source_name
@@ -1425,6 +1464,7 @@ class ResourceService:
source_name=plan.source_identity.source_name,
to_is_directory=planned_to_is_directory,
args=plan.processor_args,
+ git_head_sha_hint=plan.source_identity.head_sha,
defer_target_resolution=defer_target_resolution,
cleanup_empty_target_on_failure=cleanup_empty_target_on_failure,
understanding_response_id=plan.understanding_response_id,
@@ -1565,19 +1605,44 @@ class ResourceService:
source: str,
*,
auth_config: Optional[GitHttpAuthConfig] = None,
+ branch: Optional[str] = None,
+ commit: Optional[str] = None,
) -> _ResourceSourceInfo:
+ # Normalize browser/tree URLs to the clone URL and resolve the tracked
+ # ref with the same logic the accessor/watch probe uses.
+ from openviking.parse.accessors.git_accessor import GitAccessor
+
+ repo_url, tracked_branch, tracked_commit = GitAccessor._resolve_probe_target(
+ source, branch=branch, commit=commit
+ ) or (source, branch, commit)
+
+ # Target the exact tracked ref so the advertised SHA can be reused by
+ # the accessor as an immutable download hint. A pinned commit cannot be
+ # proven reachable by ls-remote, so HEAD only verifies repository
+ # access there; the fetch/checkout validates the exact SHA.
+ if tracked_commit:
+ ref = "HEAD"
+ ls_remote_args = ["git", "ls-remote", repo_url, "HEAD"]
+ elif tracked_branch:
+ ref = (
+ tracked_branch
+ if tracked_branch.startswith("refs/")
+ else f"refs/heads/{tracked_branch}"
+ )
+ ls_remote_args = ["git", "ls-remote", "--heads", repo_url, ref]
+ else:
+ ref = "HEAD"
+ ls_remote_args = ["git", "ls-remote", repo_url, "HEAD"]
+
proc = None
try:
- env = build_git_http_auth_env(auth_config, source) if auth_config is not None else None
+ env = build_git_http_auth_env(auth_config, repo_url) if auth_config is not None else None
env = overlay_parser_subprocess_env(
- ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, source),
+ ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, repo_url),
env,
)
proc = await asyncio.create_subprocess_exec(
- "git",
- "ls-remote",
- "--heads",
- source,
+ *ls_remote_args,
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
@@ -1606,11 +1671,21 @@ class ResourceService:
if proc.returncode != 0:
raise_git_auth_error(stderr)
raise InvalidArgumentError("Cannot access Git repository; git ls-remote failed.")
+ # Best-effort SHA extraction only: an empty repository or a ref the
+ # server does not advertise still exits 0 and must pass preflight. In
+ # that case the hint stays unset and the clone path performs the
+ # definitive ref/error classification.
+ head_sha = None
+ if not tracked_commit:
+ head_sha = GitAccessor._parse_ls_remote_ref(
+ stdout.decode(errors="replace"), ref
+ )
repo_name = parse_code_hosting_url(source)
return _ResourceSourceInfo(
source_name=repo_name.rsplit("/", 1)[-1] if repo_name else None,
source_path=source,
source_format="repository",
+ head_sha=head_sha,
)
async def add_resource(
@@ -2525,6 +2600,7 @@ class ResourceService:
user_id=ctx.user.user_id,
role=str(ctx.role),
is_active=False,
+ invalidate_source_checkpoint=True,
)
logger.info(
f"[ResourceService] Deactivated watch task {existing_task.task_id} for {to_uri}"
diff --git a/openviking/storage/queuefs/add_resource_msg.py b/openviking/storage/queuefs/add_resource_msg.py
index 43e1a53d6..adcff2892 100644
--- a/openviking/storage/queuefs/add_resource_msg.py
+++ b/openviking/storage/queuefs/add_resource_msg.py
@@ -53,6 +53,10 @@ class AddResourceMsg:
watch_interval: float = 0
is_active: Optional[bool] = None
watch_task_id: Optional[str] = None
+ # Transient exact-revision hint resolved by the add_resource git
+ # preflight. Consumed by the accessor to skip a second ls-remote; it is
+ # never persisted into WatchTask processor kwargs.
+ git_head_sha_hint: Optional[str] = None
skip_watch_management: bool = True
defer_target_resolution: bool = False
cleanup_empty_target_on_failure: bool = False
@@ -214,6 +218,12 @@ class AddResourceMsg:
if isinstance(data.get("watch_task_id"), str) and data["watch_task_id"]
else None
),
+ git_head_sha_hint=(
+ str(data["git_head_sha_hint"])
+ if isinstance(data.get("git_head_sha_hint"), str)
+ and data["git_head_sha_hint"].strip()
+ else None
+ ),
skip_watch_management=bool(data.get("skip_watch_management", True)),
defer_target_resolution=bool(data.get("defer_target_resolution", False)),
cleanup_empty_target_on_failure=bool(
diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.py
index a54bde19c..27fe29e0d 100644
--- a/openviking/storage/queuefs/add_resource_processor.py
+++ b/openviking/storage/queuefs/add_resource_processor.py
@@ -110,15 +110,28 @@ class AddResourceProcessor(DequeueHandlerBase):
msg: AddResourceMsg,
status: str,
error: Optional[str] = None,
+ result: Optional[Dict[str, Any]] = None,
) -> None:
if not msg.watch_task_id:
return
try:
+ source_checkpoint = (
+ self._resource_service._initial_source_checkpoint(
+ result.get("meta"),
+ instruction=msg.instruction,
+ )
+ if status == "completed" and isinstance(result, dict)
+ else None
+ )
+ kwargs = {}
+ if source_checkpoint is not None:
+ kwargs["source_checkpoint"] = source_checkpoint
await self._resource_service.record_watch_execution(
msg.watch_task_id,
status=status,
execution_task_id=msg.task_id,
error=error,
+ **kwargs,
)
except Exception:
logger.exception("[AddResource] Failed to record initial Watch execution")
@@ -359,7 +372,7 @@ class AddResourceProcessor(DequeueHandlerBase):
reason=msg.reason,
source_name=msg.source_name,
)
- await self._record_watch_execution(msg, "completed")
+ await self._record_watch_execution(msg, "completed", result=result)
await tracker.complete(
msg.task_id,
result,
diff --git a/tests/parse/accessors/test_git_accessor_probe.py b/tests/parse/accessors/test_git_accessor_probe.py
new file mode 100644
index 000000000..3acb4539b
--- /dev/null
+++ b/tests/parse/accessors/test_git_accessor_probe.py
@@ -0,0 +1,241 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Tests for the lightweight git ls-remote watch probe."""
+
+import asyncio
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from openviking.parse.accessors import git_accessor
+from openviking.parse.accessors.git_accessor import GitAccessor
+from openviking.parse.network import (
+ ParserNetworkOperation,
+ register_parser_network_adapter,
+ reset_parser_network_adapter,
+)
+
+_resolve_probe_target = GitAccessor._resolve_probe_target
+_parse_ls_remote_ref = GitAccessor._parse_ls_remote_ref
+
+
+def probe_git_remote_head(source, **kwargs):
+ return GitAccessor().probe_git_remote_head(source, **kwargs)
+
+
+class FakeProc:
+ def __init__(self, returncode=0, stdout=b"", stderr=b""):
+ self.returncode = returncode
+ self.pid = 12345
+ self._stdout = stdout
+ self._stderr = stderr
+ self.calls = []
+
+ async def communicate(self):
+ self.calls.append("communicate")
+ return self._stdout, self._stderr
+
+
+class TestProbeTargetResolution:
+ def test_explicit_branch_kept_for_plain_clone_url(self):
+ url, branch, commit = _resolve_probe_target(
+ "https://github.com/volcengine/OpenViking.git", branch="dev"
+ )
+ assert url == "https://github.com/volcengine/OpenViking.git"
+ assert branch == "dev"
+ assert commit is None
+
+ def test_explicit_commit_wins_over_branch(self):
+ url, branch, commit = _resolve_probe_target(
+ "https://github.com/volcengine/OpenViking.git",
+ branch="main",
+ commit="a" * 40,
+ )
+ assert url == "https://github.com/volcengine/OpenViking.git"
+ assert branch is None
+ assert commit == "a" * 40
+
+ def test_tree_url_branch_is_resolved(self):
+ url, branch, commit = _resolve_probe_target(
+ "https://github.com/volcengine/OpenViking/tree/release-1.0"
+ )
+ assert url == "https://github.com/volcengine/OpenViking"
+ assert branch == "release-1.0"
+ assert commit is None
+
+ def test_commit_pin_short_circuits(self):
+ url, branch, commit = _resolve_probe_target(
+ "https://github.com/volcengine/OpenViking/commit/abcdef1234567890"
+ )
+ assert commit == "abcdef1234567890"
+ assert branch is None
+
+ def test_explicit_commit_overrides_url_commit(self):
+ url, branch, commit = _resolve_probe_target(
+ "https://github.com/volcengine/OpenViking/commit/abcdef1234567890",
+ commit="deadbeef",
+ )
+ assert url == "https://github.com/volcengine/OpenViking"
+ assert branch is None
+ assert commit == "deadbeef"
+
+ def test_ssh_url_passthrough(self):
+ source = "[email protected]:volcengine/OpenViking.git"
+ url, branch, commit = _resolve_probe_target(source, ref="main")
+ assert url == source
+ assert branch == "main"
+
+ def test_non_git_source_returns_none(self, tmp_path):
+ assert _resolve_probe_target(str(tmp_path / "repo")) is None
+
+
+class TestLsRemoteOutputParsers:
+ def test_heads_output(self):
+ stdout = f"{'a' * 40}\trefs/heads/main\n{'b' * 40}\trefs/heads/dev\n"
+ assert _parse_ls_remote_ref(stdout, "refs/heads/main") == "a" * 40
+ assert _parse_ls_remote_ref(stdout, "refs/heads/release") is None
+
+ def test_head_output(self):
+ stdout = f"{'c' * 40}\tHEAD\n"
+ assert _parse_ls_remote_ref(stdout, "HEAD") == "c" * 40
+ assert _parse_ls_remote_ref(stdout, "refs/heads/main") is None
+
+
+class TestProbeGitRemoteHead:
+ @pytest.mark.asyncio
+ async def test_branch_without_hint_downloads_by_name_without_checkpoint(
+ self, monkeypatch
+ ):
+ # Without a commit pin or git_head_sha_hint the accessor must NOT issue
+ # its own ls-remote: it downloads the moving branch by name and records
+ # no exact-revision checkpoint (fail-open to the legacy path).
+ accessor = GitAccessor()
+ local_dir = Path("/tmp/openviking-test-repository")
+ zip_download = AsyncMock(return_value=(local_dir, "volcengine/OpenViking"))
+ monkeypatch.setattr(accessor, "_github_zip_download", zip_download)
+ monkeypatch.setattr(
+ accessor,
+ "probe_git_remote_head",
+ AsyncMock(side_effect=AssertionError("accessor must not self-probe")),
+ )
+
+ resource = await accessor.access(
+ "https://github.com/volcengine/OpenViking.git",
+ branch="main",
+ )
+
+ assert zip_download.await_args.args[1] == "main"
+ assert resource.path == local_dir
+ assert resource.meta.get("resolved_head_sha") is None
+
+ @pytest.mark.asyncio
+ async def test_probe_applies_parser_network_adapter(self, monkeypatch):
+ accessor = GitAccessor()
+ run_git = AsyncMock(return_value=f"{'a' * 40}\trefs/heads/main\n")
+ monkeypatch.setattr(accessor, "_run_git", run_git)
+ register_parser_network_adapter(
+ SimpleNamespace(
+ supports=lambda context: context.operation is ParserNetworkOperation.GIT_COMMAND,
+ subprocess_env_overlay=lambda _context: {"OV_GIT_ROUTE": "adapter"},
+ )
+ )
+ try:
+ result = await accessor.probe_git_remote_head(
+ "https://github.com/volcengine/OpenViking.git",
+ branch="main",
+ )
+ finally:
+ reset_parser_network_adapter()
+
+ assert result == ("refs/heads/main", "a" * 40)
+ assert run_git.await_args.kwargs["env"]["OV_GIT_ROUTE"] == "adapter"
+
+ @pytest.mark.asyncio
+ async def test_known_branch_uses_heads_refspec(self, monkeypatch):
+ captured = {}
+ fake_proc = FakeProc(stdout=f"{'a' * 40}\trefs/heads/main\n".encode())
+
+ async def fake_exec(*args, **kwargs):
+ captured["args"] = args
+ captured["kwargs"] = kwargs
+ return fake_proc
+
+ monkeypatch.setattr(git_accessor.asyncio, "create_subprocess_exec", fake_exec)
+
+ ref, sha = await probe_git_remote_head(
+ "https://github.com/volcengine/OpenViking.git", branch="main"
+ )
+
+ assert ref == "refs/heads/main"
+ assert sha == "a" * 40
+ git_args = captured["args"]
+ assert git_args[:3] == ("git", "ls-remote", "--heads")
+ assert git_args[-1] == "refs/heads/main"
+
+ @pytest.mark.asyncio
+ async def test_unknown_branch_tracks_remote_head(self, monkeypatch):
+ captured = {}
+ fake_proc = FakeProc(stdout=f"{'d' * 40}\tHEAD\n".encode())
+
+ async def fake_exec(*args, **kwargs):
+ captured["args"] = args
+ return fake_proc
+
+ monkeypatch.setattr(git_accessor.asyncio, "create_subprocess_exec", fake_exec)
+
+ ref, sha = await probe_git_remote_head(
+ "https://github.com/volcengine/OpenViking.git"
+ )
+
+ assert (ref, sha) == ("HEAD", "d" * 40)
+ git_args = captured["args"]
+ assert git_args[:3] == ("git", "ls-remote", "https://github.com/volcengine/OpenViking.git")
+ assert git_args[-1] == "HEAD"
+ assert "--symref" not in git_args
+
+ @pytest.mark.asyncio
+ async def test_commit_pin_returns_supplied_commit_without_subprocess(self, monkeypatch):
+ async def fake_exec(*_args, **_kwargs):
+ raise AssertionError("must not spawn git for a commit pin")
+
+ monkeypatch.setattr(git_accessor.asyncio, "create_subprocess_exec", fake_exec)
+
+ result = await probe_git_remote_head(
+ "https://github.com/volcengine/OpenViking/commit/abcdef1234567890"
+ )
+ assert result == ("", "abcdef1234567890")
+
+ @pytest.mark.asyncio
+ async def test_non_git_source_returns_none(self, monkeypatch, tmp_path):
+ result = await probe_git_remote_head(str(tmp_path / "repo"))
+ assert result is None
+
+ @pytest.mark.asyncio
+ async def test_git_failure_raises(self, monkeypatch):
+ fake_proc = FakeProc(returncode=1, stderr=b"fatal: not found")
+
+ async def fake_exec(*_args, **_kwargs):
+ return fake_proc
+
+ monkeypatch.setattr(git_accessor.asyncio, "create_subprocess_exec", fake_exec)
+
+ with pytest.raises(RuntimeError):
+ await probe_git_remote_head(
+ "https://github.com/volcengine/OpenViking.git", branch="main"
+ )
+
+ @pytest.mark.asyncio
+ async def test_missing_ref_in_output_raises(self, monkeypatch):
+ fake_proc = FakeProc(stdout=b"")
+
+ async def fake_exec(*_args, **_kwargs):
+ return fake_proc
+
+ monkeypatch.setattr(git_accessor.asyncio, "create_subprocess_exec", fake_exec)
+
+ with pytest.raises(RuntimeError, match="did not advertise"):
+ await probe_git_remote_head(
+ "https://github.com/volcengine/OpenViking.git", branch="ghost"
+ )
diff --git a/tests/parse/accessors/test_git_accessor_process_cleanup.py b/tests/parse/accessors/test_git_accessor_process_cleanup.py
new file mode 100644
index 000000000..d5cf40f9d
--- /dev/null
+++ b/tests/parse/accessors/test_git_accessor_process_cleanup.py
@@ -0,0 +1,138 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Real-subprocess regression tests for git timeout cancellation.
+
+These tests deliberately do NOT mock the subprocess layer. They install a
+real ``git-remote-block`` executable on PATH that spawns a long-lived child
+and blocks on the helper protocol, reproducing the case where git's remote
+helper (and its descendant) survive a kill of the direct git child and keep
+the stdout/stderr pipes and the process tree occupied.
+"""
+
+import asyncio
+import contextlib
+import os
+import shutil
+import signal
+import time
+
+import pytest
+
+from openviking.parse.accessors.git_accessor import GitAccessor
+
+pytestmark = pytest.mark.skipif(
+ os.name != "posix" or shutil.which("git") is None,
+ reason="process-group cleanup requires POSIX and a real git binary",
+)
+
+# Cancellation must propagate plus complete the bounded group reap well within
+# this window; without the fix the helper descendants linger for 30 seconds.
+CANCEL_BOUND_SECONDS = 5.0
+
+# A minimal blocking remote helper:
+# 1. forks a ``sleep`` descendant that stays in the same process group;
+# 2. records helper/child PIDs into a marker file;
+# 3. blocks forever reading git's command stream (like a wedged helper).
+_BLOCKING_HELPER = """#!/bin/sh
+target="${1#block::}"
+sleep 30 &
+child=$!
+printf 'helper=%s\\nchild=%s\\n' "$$" "$child" >> "$target"
+while read -r _line; do :; done
+wait "$child"
+"""
+
+
+def _pid_alive(pid: int) -> bool:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return False
+ except PermissionError:
+ return True
+ return True
+
+
+async def _wait_for_marker(marker, timeout: float = 5.0) -> None:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if marker.exists():
+ return
+ await asyncio.sleep(0.02)
+ raise AssertionError(f"remote helper never started, marker missing: {marker}")
+
+
+async def _wait_until_dead(pid: int, timeout: float = 2.0) -> None:
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if not _pid_alive(pid):
+ return
+ await asyncio.sleep(0.02)
+ raise AssertionError(f"process {pid} still alive after cancellation")
+
+
+async def test_blocking_remote_helper_tree_is_killed_on_timeout(tmp_path):
+ bin_dir = tmp_path / "bin"
+ bin_dir.mkdir()
+ marker = tmp_path / "helper.pids"
+ helper = bin_dir / "git-remote-block"
+ helper.write_text(_BLOCKING_HELPER, encoding="utf-8")
+ helper.chmod(0o755)
+
+ # An isolated environment so user/system git config cannot interfere and
+ # git actually resolves our helper through PATH.
+ env = dict(os.environ)
+ env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}"
+ env["HOME"] = str(tmp_path)
+ env["GIT_TERMINAL_PROMPT"] = "0"
+ env["GIT_CONFIG_NOSYSTEM"] = "1"
+
+ git_args = [
+ "git",
+ "-c",
+ "protocol.block.allow=always",
+ "ls-remote",
+ f"block::{marker}",
+ ]
+
+ accessor = GitAccessor()
+ # Start the command first so we can observe the real process tree before
+ # cancellation.
+ task = asyncio.ensure_future(accessor._run_git(git_args, env=env))
+ helper_pid = None
+ child_pid = None
+ try:
+ await _wait_for_marker(marker)
+ for line in marker.read_text(encoding="utf-8").splitlines():
+ key, _, value = line.partition("=")
+ if key == "helper":
+ helper_pid = int(value)
+ elif key == "child":
+ child_pid = int(value)
+ assert helper_pid is not None and child_pid is not None
+
+ # The whole tree (git -> helper -> sleep child) must exist and block.
+ assert _pid_alive(helper_pid)
+ assert _pid_alive(child_pid)
+
+ # Timeout cancels _run_git; cleanup must kill the process group and
+ # return promptly instead of waiting for the 30s sleep descendant.
+ start = time.monotonic()
+ with pytest.raises(asyncio.TimeoutError):
+ await asyncio.wait_for(task, timeout=0.5)
+ elapsed = time.monotonic() - start
+
+ # Helper and its descendant are SIGKILLed together with the group.
+ await _wait_until_dead(helper_pid)
+ await _wait_until_dead(child_pid)
+ assert elapsed < CANCEL_BOUND_SECONDS
+ finally:
+ # Never leak helper processes if an assertion above failed.
+ for pid in (helper_pid, child_pid):
+ if pid is not None and _pid_alive(pid):
+ with contextlib.suppress(ProcessLookupError):
+ os.kill(pid, signal.SIGKILL)
+ if not task.done():
+ task.cancel()
+ with contextlib.suppress(Exception):
+ await task
diff --git a/tests/resource/test_watch_scheduler.py b/tests/resource/test_watch_scheduler.py
index 9d5b05a9e..cce50cd6e 100644
--- a/tests/resource/test_watch_scheduler.py
+++ b/tests/resource/test_watch_scheduler.py
@@ -1,6 +1,7 @@
import asyncio
import threading
from datetime import datetime, timedelta
+from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
@@ -385,3 +386,303 @@ class TestWatchSchedulerResourceExistence:
assert updated.last_task_id == "add-resource-task-1"
assert updated.last_status == "failed"
assert updated.last_error == "document import failed"
+
+
+class TestWatchSchedulerGitShaGate:
+ REPO = "https://github.com/volcengine/OpenViking.git"
+
+ def _build_scheduler(self, monkeypatch, probe):
+ monkeypatch.setattr(
+ "openviking.parse.accessors.git_accessor.GitAccessor.probe_git_remote_head",
+ probe,
+ )
+
+ class FakeResourceService(ResourceService):
+ def __init__(self):
+ super().__init__()
+ self.calls = []
+
+ async def refresh_resource(self, **kwargs):
+ self.calls.append(kwargs)
+ return {"root_uri": kwargs.get("to"), "status": "completed"}
+
+ resource_service = FakeResourceService()
+ scheduler = WatchScheduler(resource_service=resource_service, check_interval=1)
+ return scheduler, resource_service
+
+ async def _setup_task(
+ self,
+ scheduler,
+ *,
+ source_checkpoint=None,
+ path=None,
+ processor_kwargs=None,
+ instruction="",
+ ):
+ manager = WatchManager(viking_fs=None)
+ await manager.initialize()
+ scheduler._watch_manager = manager
+ task = await manager.create_task(
+ path=path or self.REPO,
+ to_uri="viking://resources/openviking",
+ watch_interval=30.0,
+ processor_kwargs=processor_kwargs,
+ instruction=instruction,
+ )
+ if source_checkpoint is not None:
+ await manager.update_source_checkpoint(task.task_id, source_checkpoint)
+ return manager, task
+
+ @pytest.mark.asyncio
+ async def test_unchanged_head_sha_skips_refresh(self, monkeypatch):
+ state = {"kind": "git", "head_sha": "a" * 40, "instruction": ""}
+ probe = AsyncMock(return_value=("refs/heads/main", "a" * 40))
+ scheduler, resource_service = self._build_scheduler(monkeypatch, probe)
+ manager, task = await self._setup_task(scheduler, source_checkpoint=state)
+
+ await scheduler._execute_task(task)
+
+ probe.assert_awaited_once()
+ assert resource_service.calls == []
+ updated = await manager.get_task(task.task_id)
+ assert updated.last_status == "completed"
+ assert updated.source_checkpoint == state
+ assert updated.next_execution_time is not None
+ assert updated.next_execution_time > datetime.now()
+
+ @pytest.mark.asyncio
+ async def test_unchanged_skip_preserves_last_task_id(self, monkeypatch):
+ state = {"kind": "git", "head_sha": "a" * 40, "instruction": ""}
+ probe = AsyncMock(return_value=("refs/heads/main", "a" * 40))
+ scheduler, resource_service = self._build_scheduler(monkeypatch, probe)
+ manager, task = await self._setup_task(scheduler, source_checkpoint=state)
+ await manager.record_execution(
+ task.task_id,
+ status="completed",
+ execution_task_id="ingest-task-1",
+ )
+
+ await scheduler._execute_task(task)
+
+ assert resource_service.calls == []
+ updated = await manager.get_task(task.task_id)
+ assert updated.last_status == "completed"
+ assert updated.last_task_id == "ingest-task-1"
+
+ @pytest.mark.asyncio
+ async def test_commit_pin_uses_commit_checkpoint_without_refresh(self, monkeypatch):
+ commit = "a" * 40
+ probe = AsyncMock(return_value=("", commit))
+ scheduler, resource_service = self._build_scheduler(monkeypatch, probe)
+ manager, task = await self._setup_task(
+ scheduler,
+ source_checkpoint={"kind": "git", "head_sha": commit, "instruction": ""},
+ processor_kwargs={"commit": commit},
+ )
+
+ await scheduler._execute_task(task)
+
+ probe.assert_awaited_once()
+ assert resource_service.calls == []
+ updated = await manager.get_task(task.task_id)
+ assert updated.last_status == "completed"
+ assert updated.source_checkpoint == {
+ "kind": "git",
+ "head_sha": commit,
+ "instruction": "",
+ }
+
+ @pytest.mark.asyncio
+ async def test_account_github_token_is_used_for_probe_without_persisting(self, monkeypatch):
+ state = {"kind": "git", "head_sha": "a" * 40, "instruction": ""}
+ probe = AsyncMock(return_value=("refs/heads/main", "a" * 40))
+ scheduler, resource_service = self._build_scheduler(monkeypatch, probe)
+ token_resolver = AsyncMock(return_value="account-token")
+ resource_service._resource_processor = SimpleNamespace(
+ github_token_for=token_resolver,
+ )
+ manager, task = await self._setup_task(scheduler, source_checkpoint=state)
+
+ await scheduler._execute_task(task)
+
+ token_resolver.assert_awaited_once()
+ probe.assert_awaited_once_with(
+ self.REPO,
+ auth_config={"token": "account-token"},
+ )
+ assert resource_service.calls == []
+ updated = await manager.get_task(task.task_id)
+ assert updated.source_checkpoint == state
+ assert "auth_config" not in updated.processor_kwargs
+
+ @pytest.mark.asyncio
+ async def test_changed_instruction_triggers_refresh(self, monkeypatch):
+ sha = "a" * 40
+ probe = AsyncMock(return_value=("refs/heads/main", sha))
+ scheduler, resource_service = self._build_scheduler(monkeypatch, probe)
+ manager, task = await self._setup_task(
+ scheduler,
+ source_checkpoint={
+ "kind": "git",
+ "head_sha": sha,
+ "instruction": "old instruction",
+ },
+ instruction="new instruction",
+ )
+
+ await scheduler._execute_task(task)
+
+ assert len(resource_service.calls) == 1
+ assert resource_service.calls[0]["instruction"] == "new instruction"
+ updated = await manager.get_task(task.task_id)
+ assert updated.last_status == "completed"
+
+ @pytest.mark.asyncio
+ async def test_probe_timeout_fails_open_to_refresh(self, monkeypatch):
+ async def blocked_probe(*_args, **_kwargs):
+ await asyncio.Event().wait()
+
+ scheduler, resource_service = self._build_scheduler(monkeypatch, blocked_probe)
+ monkeypatch.setattr(scheduler, "DEFAULT_GIT_PROBE_TIMEOUT", 0.01)
+ manager, task = await self._setup_task(scheduler)
+
+ await asyncio.wait_for(scheduler._execute_task(task), timeout=1.0)
+
+ assert len(resource_service.calls) == 1
+ updated = await manager.get_task(task.task_id)
+ assert updated.last_status == "completed"
+
+ @pytest.mark.asyncio
+ async def test_changed_head_sha_triggers_refresh(self, monkeypatch):
+ probe = AsyncMock(return_value=("refs/heads/main", "b" * 40))
+ scheduler, resource_service = self._build_scheduler(monkeypatch, probe)
+ manager, task = await self._setup_task(scheduler)
+
+ await scheduler._execute_task(task)
+
+ assert len(resource_service.calls) == 1
+ updated = await manager.get_task(task.task_id)
+ assert updated.source_checkpoint is None
+ assert updated.last_status == "completed"
+
+ @pytest.mark.asyncio
+ async def test_async_refresh_uses_completed_result_checkpoint(self, monkeypatch):
+ probe = AsyncMock(return_value=("refs/heads/main", "b" * 40))
+ monkeypatch.setattr(
+ "openviking.parse.accessors.git_accessor.GitAccessor.probe_git_remote_head",
+ probe,
+ )
+
+ class FakeResourceService(ResourceService):
+ async def refresh_resource(self, **_kwargs):
+ return {"status": "success", "task_id": "refresh-task-1"}
+
+ class CompletedStatus:
+ value = "completed"
+
+ class CompletedTask:
+ status = CompletedStatus()
+ error = None
+ result = {"meta": {"resolved_head_sha": "b" * 40}}
+
+ tracker = AsyncMock()
+ tracker.wait.return_value = CompletedTask()
+ monkeypatch.setattr(
+ "openviking.service.task_tracker.get_task_tracker",
+ lambda: tracker,
+ )
+ scheduler = WatchScheduler(
+ resource_service=FakeResourceService(),
+ check_interval=1,
+ )
+ manager, task = await self._setup_task(
+ scheduler,
+ source_checkpoint={
+ "kind": "git",
+ "head_sha": "b" * 40,
+ "instruction": "old instruction",
+ },
+ instruction="new instruction",
+ )
+
+ await scheduler._execute_task(task)
+
+ updated = await manager.get_task(task.task_id)
+ assert updated.source_checkpoint == {
+ "kind": "git",
+ "head_sha": "b" * 40,
+ "instruction": "new instruction",
+ }
+ assert updated.last_status == "completed"
+
+ @pytest.mark.asyncio
+ async def test_probe_failure_fails_open_to_full_refresh(self, monkeypatch):
+ async def failing_probe(*_args, **_kwargs):
+ raise RuntimeError("ls-remote boom")
+
+ scheduler, resource_service = self._build_scheduler(monkeypatch, failing_probe)
+ manager, task = await self._setup_task(
+ scheduler,
+ source_checkpoint={"kind": "git", "head_sha": "a" * 40},
+ )
+
+ await scheduler._execute_task(task)
+
+ assert len(resource_service.calls) == 1
+ updated = await manager.get_task(task.task_id)
+ assert updated.last_status == "completed"
+ # No fresh checkpoint available: previous state is left untouched.
+ assert updated.source_checkpoint["head_sha"] == "a" * 40
+
+ @pytest.mark.asyncio
+ async def test_sync_completed_refresh_persists_checkpoint(self, monkeypatch):
+ sha = "c" * 40
+ probe = AsyncMock(return_value=("refs/heads/main", sha))
+ monkeypatch.setattr(
+ "openviking.parse.accessors.git_accessor.GitAccessor.probe_git_remote_head",
+ probe,
+ )
+
+ class SyncResourceService(ResourceService):
+ def __init__(self):
+ super().__init__()
+ self.calls = []
+
+ async def refresh_resource(self, **kwargs):
+ self.calls.append(kwargs)
+ # No queue manager: the terminal result is returned directly,
+ # carrying the imported revision meta without a task_id.
+ return {
+ "root_uri": kwargs.get("to"),
+ "status": "completed",
+ "meta": {"resolved_head_sha": sha},
+ }
+
+ resource_service = SyncResourceService()
+ scheduler = WatchScheduler(resource_service=resource_service, check_interval=1)
+ manager, task = await self._setup_task(scheduler)
+
+ await scheduler._execute_task(task)
+
+ assert len(resource_service.calls) == 1
+ updated = await manager.get_task(task.task_id)
+ assert updated.last_status == "completed"
+ assert updated.source_checkpoint == {
+ "kind": "git",
+ "head_sha": sha,
+ "instruction": "",
+ }
+
+ @pytest.mark.asyncio
+ async def test_non_git_source_does_not_probe(self, monkeypatch):
+ probe = AsyncMock(return_value=("refs/heads/main", "a" * 40))
+ scheduler, resource_service = self._build_scheduler(monkeypatch, probe)
+ manager, task = await self._setup_task(
+ scheduler,
+ path="https://example.com/doc.md",
+ )
+
+ await scheduler._execute_task(task)
+
+ probe.assert_not_awaited()
+ assert len(resource_service.calls) == 1
diff --git a/tests/service/test_resource_service_connector.py b/tests/service/test_resource_service_connector.py
index c32895f1f..3014d6850 100644
--- a/tests/service/test_resource_service_connector.py
+++ b/tests/service/test_resource_service_connector.py
@@ -1910,6 +1910,43 @@ async def test_native_git_enqueue_rejects_credentials_before_durable_job(
service._enqueue_add_resource_job.assert_not_awaited()
+async def test_native_git_preflight_sha_is_carried_as_transient_msg_hint(
+ monkeypatch,
+ connector_config,
+ ctx,
+ service,
+):
+ monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
+ service._connector.should_delegate = Mock(return_value=False)
+ head_sha = "e" * 40
+ service._preflight_git_source = AsyncMock(
+ return_value=SimpleNamespace(
+ source_name="private",
+ source_path="https://git.example/org/private.git",
+ source_format="repository",
+ head_sha=head_sha,
+ )
+ )
+ service._plan_source_job_target = AsyncMock(
+ return_value=("viking://resources/private", None, False, False)
+ )
+ service._enqueue_add_resource_job = AsyncMock(
+ return_value=SimpleNamespace(task_id="task-private")
+ )
+
+ await service.add_resource(
+ path="https://git.example/org/private.git",
+ ctx=ctx,
+ to="viking://resources/private",
+ )
+
+ message = service._enqueue_add_resource_job.await_args.args[0]
+ assert message.git_head_sha_hint == head_sha
+ # Transient hint survives durable serialization for the worker.
+ assert AddResourceMsg.from_dict(message.to_dict()).git_head_sha_hint == head_sha
+
+
@pytest.mark.asyncio
async def test_native_git_nested_auth_is_owned_by_durable_task(
monkeypatch,
@@ -1924,6 +1961,7 @@ async def test_native_git_nested_auth_is_owned_by_durable_task(
source_name="private",
source_path="https://git.example/org/private.git",
source_format="repository",
+ head_sha=None,
)
)
service._plan_source_job_target = AsyncMock(
@@ -1973,6 +2011,7 @@ async def test_native_git_nested_auth_watch_is_copied_from_task_state_by_worker(
source_name="private",
source_path="https://git.example/org/private.git",
source_format="repository",
+ head_sha=None,
)
)
service._plan_source_job_target = AsyncMock(
@@ -2016,6 +2055,7 @@ async def test_native_git_watch_refresh_queues_with_restored_task_auth(
source_name="private",
source_path="https://git.example/org/private.git",
source_format="repository",
+ head_sha=None,
)
)
service._plan_source_job_target = AsyncMock(
@@ -2174,6 +2214,7 @@ async def test_add_resource_falls_back_for_shared_source_with_parent(
source_name="repo",
source_path="https://git.example/org/repo.git",
source_format="repository",
+ head_sha=None,
)
)
service._plan_source_job_target = AsyncMock(
@@ -2231,6 +2272,7 @@ async def test_add_resource_job_executes_frozen_route(ctx, service):
)
resource_lock = SimpleNamespace(close=AsyncMock())
stage_callback = AsyncMock()
+ head_sha = "f" * 40
msg = AddResourceMsg(
task_id="task-1",
path="git://example.com/repo.git",
@@ -2239,6 +2281,7 @@ async def test_add_resource_job_executes_frozen_route(ctx, service):
user_id=ctx.user.user_id,
role=str(ctx.role),
args={"parser_backend": "understanding"},
+ git_head_sha_hint=head_sha,
)
result = await service.execute_add_resource_job(
@@ -2255,6 +2298,7 @@ async def test_add_resource_job_executes_frozen_route(ctx, service):
assert call["defer_post_processing"] is False
assert call["resource_lock"] is resource_lock
assert call["parser_backend"] == "understanding"
+ assert call["git_head_sha_hint"] == head_sha
@pytest.mark.asyncio
diff --git a/tests/service/test_resource_service_git_preflight.py b/tests/service/test_resource_service_git_preflight.py
index da91054e1..580960734 100644
--- a/tests/service/test_resource_service_git_preflight.py
+++ b/tests/service/test_resource_service_git_preflight.py
@@ -31,7 +31,8 @@ async def test_account_token_and_parser_proxy_reach_repository_preflight(monkeyp
processor = SimpleNamespace(github_token_for=token_resolver)
service = ResourceService(resource_processor=processor)
process = Mock(returncode=0)
- process.communicate = AsyncMock(return_value=(b"refs/heads/main\n", b""))
+ head_sha = "a" * 40
+ process.communicate = AsyncMock(return_value=(f"{head_sha}\tHEAD\n".encode(), b""))
execute = AsyncMock(return_value=process)
monkeypatch.setattr(asyncio, "create_subprocess_exec", execute)
monkeypatch.setattr(
@@ -66,8 +67,8 @@ async def test_account_token_and_parser_proxy_reach_repository_preflight(monkeyp
assert execute.await_args.args == (
"git",
"ls-remote",
- "--heads",
"https://github.com/org/private",
+ "HEAD",
)
env = execute.await_args.kwargs["env"]
assert env["HTTPS_PROXY"] == "http://proxy.example:8118"
@@ -80,6 +81,7 @@ async def test_account_token_and_parser_proxy_reach_repository_preflight(monkeyp
f"Authorization: Basic {encoded}"
)
assert plan is not None
+ assert plan.source_identity.head_sha == head_sha
assert plan.task_auth == {}
assert "account-token" not in str(plan.processor_args)
@@ -100,6 +102,7 @@ async def test_default_github_token_is_not_applied_to_ssh_preflight(monkeypatch,
source_name="private",
source_path=source,
source_format="repository",
+ head_sha=None,
)
)
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
@@ -119,7 +122,9 @@ async def test_default_github_token_is_not_applied_to_ssh_preflight(monkeypatch,
)
token_resolver.assert_not_awaited()
- service._preflight_git_source.assert_awaited_once_with(source, auth_config=None)
+ service._preflight_git_source.assert_awaited_once_with(
+ source, auth_config=None, branch=None, commit=None
+ )
assert plan is not None
@@ -187,3 +192,92 @@ async def test_git_preflight_cancellation_kills_process_group_and_bounds_reap(mo
wait_timeouts,
) == (True, [call(process.pid, signal.SIGKILL)], [10.0, 1.0])
process.kill.assert_not_called()
+
+
+async def test_preflight_branched_watch_targets_ref_and_extracts_sha(monkeypatch):
+ head_sha = "b" * 40
+ process = Mock(returncode=0)
+ process.communicate = AsyncMock(
+ return_value=(f"{head_sha}\trefs/heads/release-1.0\n".encode(), b"")
+ )
+ execute = AsyncMock(return_value=process)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", execute)
+
+ info = await ResourceService()._preflight_git_source(
+ "https://github.com/org/repo.git",
+ branch="release-1.0",
+ )
+
+ assert execute.await_args.args == (
+ "git",
+ "ls-remote",
+ "--heads",
+ "https://github.com/org/repo.git",
+ "refs/heads/release-1.0",
+ )
+ assert info.head_sha == head_sha
+
+
+async def test_preflight_tree_url_branch_is_resolved_from_url(monkeypatch):
+ head_sha = "c" * 40
+ process = Mock(returncode=0)
+ process.communicate = AsyncMock(
+ return_value=(f"{head_sha}\trefs/heads/release-1.0\n".encode(), b"")
+ )
+ execute = AsyncMock(return_value=process)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", execute)
+
+ info = await ResourceService()._preflight_git_source(
+ "https://github.com/org/repo/tree/release-1.0",
+ )
+
+ assert execute.await_args.args == (
+ "git",
+ "ls-remote",
+ "--heads",
+ "https://github.com/org/repo",
+ "refs/heads/release-1.0",
+ )
+ assert info.head_sha == head_sha
+
+
+async def test_preflight_empty_repository_passes_without_sha(monkeypatch):
+ # git ls-remote exits 0 with empty output for an unborn HEAD; preflight must
+ # stay advisory and leave the hint unset instead of rejecting the import.
+ process = Mock(returncode=0)
+ process.communicate = AsyncMock(return_value=(b"", b""))
+ execute = AsyncMock(return_value=process)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", execute)
+
+ info = await ResourceService()._preflight_git_source(
+ "https://github.com/org/empty.git",
+ )
+
+ assert execute.await_args.args == (
+ "git",
+ "ls-remote",
+ "https://github.com/org/empty.git",
+ "HEAD",
+ )
+ assert info.head_sha is None
+
+
+async def test_preflight_commit_pin_checks_head_and_skips_sha_hint(monkeypatch):
+ process = Mock(returncode=0)
+ process.communicate = AsyncMock(return_value=(f"{'d' * 40}\tHEAD\n".encode(), b""))
+ execute = AsyncMock(return_value=process)
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", execute)
+
+ info = await ResourceService()._preflight_git_source(
+ "https://github.com/org/repo.git",
+ commit="deadbeef",
+ )
+
+ assert execute.await_args.args == (
+ "git",
+ "ls-remote",
+ "https://github.com/org/repo.git",
+ "HEAD",
+ )
+ # The accessor derives resolved_head_sha from the pinned commit itself.
+ assert info.head_sha is None
diff --git a/tests/service/test_resource_service_parse_mode.py b/tests/service/test_resource_service_parse_mode.py
index 033740832..64b295ff1 100644
--- a/tests/service/test_resource_service_parse_mode.py
+++ b/tests/service/test_resource_service_parse_mode.py
@@ -405,6 +405,7 @@ async def test_native_git_enqueue_persists_internal_no_split_mode(
source_name="repo",
source_path="https://github.com/volcengine/OpenViking.git",
source_format="repository",
+ head_sha=None,
)
)
service._plan_source_job_target = AsyncMock(
diff --git a/tests/service/test_resource_service_watch.py b/tests/service/test_resource_service_watch.py
index b7a0a3d7a..d84892053 100644
--- a/tests/service/test_resource_service_watch.py
+++ b/tests/service/test_resource_service_watch.py
@@ -393,6 +393,12 @@ class TestWatchTaskCreation:
def test_watch_persists_clear_without_tags(self, resource_service: ResourceService):
assert resource_service._watch_processor_kwargs({}, None, "clear") == {"tag_mode": "clear"}
+ def test_watch_kwargs_strip_transient_head_sha_hint(self, resource_service: ResourceService):
+ sanitized = resource_service._sanitize_watch_processor_kwargs(
+ {"git_head_sha_hint": "a" * 40, "branch": "main"}
+ )
+ assert sanitized == {"branch": "main"}
+
def test_add_resource_message_round_trip_preserves_clear_without_tags(self):
message = AddResourceMsg(
task_id="task-clear",
@@ -924,6 +930,7 @@ class TestAddResourceArgs:
source_name="private",
source_path=repo_url,
source_format="repository",
+ head_sha=None,
)
)
resource_service._plan_source_job_target = AsyncMock(
@@ -1047,6 +1054,7 @@ async def test_add_resource_processor_records_paused_watch_result(
service = SimpleNamespace(
execute_add_resource_job=execute,
_link_resource_reason_memory=AsyncMock(),
+ _initial_source_checkpoint=Mock(return_value=None),
record_watch_execution=AsyncMock(),
)
processor = AddResourceProcessor(
@@ -1081,6 +1089,47 @@ async def test_add_resource_processor_records_paused_watch_result(
assert task_tracker.complete.await_count == 1
+async def test_add_resource_processor_records_checkpoint_only_after_success():
+ service = SimpleNamespace(
+ _initial_source_checkpoint=ResourceService._initial_source_checkpoint,
+ record_watch_execution=AsyncMock(),
+ )
+ processor = AddResourceProcessor(
+ service,
+ QueueManager.ADD_RESOURCE,
+ SimpleNamespace(_async_agfs=SimpleNamespace()),
+ )
+ message = AddResourceMsg(
+ task_id="add-resource-1",
+ path="https://github.com/org/repo",
+ root_uri="viking://resources/repo",
+ account_id="account-1",
+ user_id="user-1",
+ role="user",
+ watch_task_id="watch-1",
+ instruction="focus on APIs",
+ )
+
+ await processor._record_watch_execution(
+ message,
+ "completed",
+ result={"meta": {"resolved_head_sha": "a" * 40}},
+ )
+
+ service.record_watch_execution.assert_awaited_once_with(
+ "watch-1",
+ status="completed",
+ execution_task_id="add-resource-1",
+ error=None,
+ source_checkpoint={
+ "kind": "git",
+ "head_sha": "a" * 40,
+ "instruction": "focus on APIs",
+ },
+ )
+
+
@pytest.mark.asyncio
async def test_record_watch_execution_redacts_error_in_log(caplog):
watch_manager = SimpleNamespace(record_execution=AsyncMock())
@@ -1115,6 +1164,34 @@ async def test_record_watch_execution_redacts_error_in_log(caplog):
scheduler.release_execution.assert_awaited_once_with("watch-1")
+async def test_record_watch_execution_persists_checkpoint_after_completed_status():
+ watch_manager = SimpleNamespace(
+ record_execution=AsyncMock(),
+ update_source_checkpoint=AsyncMock(),
+ )
+ scheduler = SimpleNamespace(
+ watch_manager=watch_manager,
+ release_execution=AsyncMock(),
+ )
+ service = ResourceService(watch_scheduler=scheduler)
+ checkpoint = {"kind": "git", "head_sha": "a" * 40}
+
+ await service.record_watch_execution(
+ "watch-1",
+ status="completed",
+ execution_task_id="add-resource-1",
+ source_checkpoint=checkpoint,
+ )
+
+ watch_manager.record_execution.assert_awaited_once()
+ watch_manager.update_source_checkpoint.assert_awaited_once_with(
+ "watch-1",
+ checkpoint,
+ )
+ scheduler.release_execution.assert_awaited_once_with("watch-1")
+
+
class TestWatchTaskConflict:
"""Tests for watch task conflict detection."""
@@ -1412,6 +1489,10 @@ class TestWatchTaskCancellation:
task = await get_task_by_uri(resource_service, to_uri, request_context)
assert task is not None
assert task.is_active is True
+ await resource_service._watch_scheduler.watch_manager.update_source_checkpoint(
+ task.task_id,
+ {"kind": "git", "head_sha": "a" * 40, "instruction": ""},
+ )
await resource_service.add_resource(
path="/test/path",
@@ -1423,6 +1504,7 @@ class TestWatchTaskCancellation:
task = await get_task_by_uri(resource_service, to_uri, request_context)
assert task is not None
assert task.is_active is False
+ assert task.source_checkpoint is None
@pytest.mark.asyncio
async def test_cancel_watch_task_with_negative_interval(
diff --git a/tests/unit/test_accessors_git.py b/tests/unit/test_accessors_git.py
index 8c28258f6..bceaafffd 100644
--- a/tests/unit/test_accessors_git.py
+++ b/tests/unit/test_accessors_git.py
@@ -5,6 +5,7 @@
import asyncio
import base64
import os
+import signal
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock, patch
@@ -331,6 +332,17 @@ class TestGitAccessor:
"1234567",
)
+ def test_parse_repo_source_explicit_commit_overrides_url_commit(
+ self, accessor: GitAccessor
+ ) -> None:
+ source = "https://github.com/org/repo/commit/abcdef1234567890"
+
+ assert accessor._parse_repo_source(source, commit="deadbeef") == (
+ "https://github.com/org/repo",
+ None,
+ "deadbeef",
+ )
+
def test_parse_repo_source_gitlab_project_named_tree(self, accessor: GitAccessor) -> None:
"""A project named tree must not shadow the /-/tree route marker."""
source = "https://gitlab.com/org/tree/-/tree/main"
@@ -539,6 +551,197 @@ class TestGitAccessor:
+ base64.b64encode(f"oauth2:{token}".encode()).decode("ascii")
)
+ async def test_archive_fallback_uses_commit_only_and_records_pin(
+ self,
+ accessor: GitAccessor,
+ tmp_path: Path,
+ ) -> None:
+ source = "https://github.com/org/repo.git"
+ commit = "a" * 40
+ with (
+ patch(
+ "openviking.parse.accessors.git_accessor.tempfile.mkdtemp",
+ return_value=str(tmp_path),
+ ),
+ patch.object(
+ accessor,
+ "_github_zip_download",
+ new=AsyncMock(side_effect=RuntimeError("archive unavailable")),
+ ) as zip_download,
+ patch.object(
+ accessor,
+ "_git_clone",
+ new=AsyncMock(return_value="org/repo"),
+ ) as git_clone,
+ patch.object(
+ accessor,
+ "_resolve_local_head",
+ new_callable=AsyncMock,
+ ) as resolve_local_head,
+ ):
+ resource = await accessor.access(
+ source,
+ branch="main",
+ commit=commit,
+ )
+
+ zip_download.assert_awaited_once_with(
+ source,
+ commit,
+ str(tmp_path),
+ github_token=None,
+ )
+ git_clone.assert_awaited_once_with(
+ source,
+ str(tmp_path),
+ branch=None,
+ commit=commit,
+ env=None,
+ )
+ resolve_local_head.assert_not_awaited()
+ assert resource.meta["resolved_head_sha"] == commit
+ assert resource.meta["repo_commit"] == commit
+ assert "repo_ref" not in resource.meta
+
+ async def test_archive_success_records_hint_sha_without_reading_local_head(
+ self,
+ accessor: GitAccessor,
+ tmp_path: Path,
+ ) -> None:
+ source = "https://github.com/org/repo.git"
+ probed_sha = "a" * 40
+ extracted = tmp_path / "repo"
+ with (
+ patch(
+ "openviking.parse.accessors.git_accessor.tempfile.mkdtemp",
+ return_value=str(tmp_path),
+ ),
+ patch.object(
+ accessor,
+ "_github_zip_download",
+ new=AsyncMock(return_value=(extracted, "org/repo")),
+ ) as zip_download,
+ patch.object(
+ accessor,
+ "_resolve_local_head",
+ new_callable=AsyncMock,
+ ) as resolve_local_head,
+ ):
+ resource = await accessor.access(
+ source,
+ branch="main",
+ git_head_sha_hint=probed_sha,
+ )
+
+ zip_download.assert_awaited_once_with(
+ source,
+ probed_sha,
+ str(tmp_path),
+ github_token=None,
+ )
+ resolve_local_head.assert_not_awaited()
+ assert resource.path == extracted
+ assert resource.meta["resolved_head_sha"] == probed_sha
+
+ async def test_github_token_archive_download_uses_hint_without_self_probe(
+ self,
+ accessor: GitAccessor,
+ tmp_path: Path,
+ ) -> None:
+ source = "https://github.com/org/private.git"
+ probed_sha = "a" * 40
+ extracted = tmp_path / "repo"
+ with (
+ patch(
+ "openviking.parse.accessors.git_accessor.tempfile.mkdtemp",
+ return_value=str(tmp_path),
+ ),
+ patch.object(
+ accessor,
+ "_git_ls_remote_head",
+ new_callable=AsyncMock,
+ ) as ls_remote,
+ patch.object(
+ accessor,
+ "_github_zip_download",
+ new=AsyncMock(return_value=(extracted, "org/private")),
+ ) as zip_download,
+ ):
+ await accessor.access(
+ source,
+ branch="main",
+ commit=None,
+ github_token="account-token",
+ git_head_sha_hint=probed_sha,
+ )
+
+ ls_remote.assert_not_awaited()
+ zip_download.assert_awaited_once_with(
+ source,
+ probed_sha,
+ str(tmp_path),
+ github_token="account-token",
+ )
+
+ async def test_commit_pin_records_supplied_commit_without_remote_probe(
+ self,
+ accessor: GitAccessor,
+ tmp_path: Path,
+ ) -> None:
+ source = "https://github.com/org/repo.git"
+ commit = "a" * 40
+ extracted = tmp_path / "repo"
+ with (
+ patch(
+ "openviking.parse.accessors.git_accessor.tempfile.mkdtemp",
+ return_value=str(tmp_path),
+ ),
+ patch.object(
+ accessor,
+ "_git_ls_remote_head",
+ new_callable=AsyncMock,
+ ) as ls_remote,
+ patch.object(
+ accessor,
+ "_github_zip_download",
+ new=AsyncMock(return_value=(extracted, "org/repo")),
+ ) as zip_download,
+ patch.object(
+ accessor,
+ "_resolve_local_head",
+ new_callable=AsyncMock,
+ ) as resolve_local_head,
+ ):
+ resource = await accessor.access(source, commit=commit)
+
+ ls_remote.assert_not_awaited()
+ zip_download.assert_awaited_once_with(
+ source,
+ commit,
+ str(tmp_path),
+ github_token=None,
+ )
+ resolve_local_head.assert_not_awaited()
+ assert resource.meta["resolved_head_sha"] == commit
+
+ async def test_commit_pin_probe_returns_supplied_commit_without_ls_remote(
+ self,
+ accessor: GitAccessor,
+ ) -> None:
+ commit = "a" * 40
+ with patch.object(
+ accessor,
+ "_git_ls_remote_head",
+ new_callable=AsyncMock,
+ ) as ls_remote:
+ probed = await accessor.probe_git_remote_head(
+ "https://github.com/org/repo.git",
+ commit=commit,
+ )
+
+ ls_remote.assert_not_awaited()
+ assert probed == ("", commit)
+
async def test_environment_github_token_does_not_replace_ssh_clone_auth(
self,
accessor: GitAccessor,
@@ -654,6 +857,21 @@ class TestGitAccessor:
assert "--no-recurse-submodules" in clone_args
assert "--recursive" not in clone_args
+ async def test_git_clone_prefers_commit_over_branch(
+ self, accessor: GitAccessor, tmp_path: Path
+ ) -> None:
+ with patch.object(accessor, "_run_git", new_callable=AsyncMock) as run_git:
+ await accessor._git_clone(
+ "https://github.com/volcengine/OpenViking.git",
+ str(tmp_path),
+ branch="main",
+ commit="deadbeef",
+ )
+
+ clone_args = run_git.await_args_list[0].args[0]
+ assert "--branch" not in clone_args
+ assert run_git.await_args_list[-1].args[0][-2:] == ["checkout", "deadbeef"]
+
async def test_git_clone_reuses_auth_env_for_clone_and_commit_fetches(
self, accessor: GitAccessor, tmp_path: Path
) -> None:
@@ -727,11 +945,12 @@ class TestGitAccessor:
assert "remote reflected" not in caplog.text
assert os.environ.get("GIT_CONFIG_COUNT") != env["GIT_CONFIG_COUNT"]
- async def test_cancelled_git_process_is_killed_and_reaped(
+ async def test_cancelled_git_process_group_is_killed_and_reaped(
self,
accessor: GitAccessor,
) -> None:
process = SimpleNamespace(
+ pid=43210,
communicate=AsyncMock(side_effect=asyncio.CancelledError),
kill=Mock(),
wait=AsyncMock(),
@@ -740,14 +959,75 @@ class TestGitAccessor:
"openviking.parse.accessors.git_accessor.asyncio.create_subprocess_exec",
new=AsyncMock(return_value=process),
):
- with pytest.raises(asyncio.CancelledError):
- await accessor._run_git(
- ["git", "clone", "https://git.example.com/org/private.git"]
- )
+ with patch(
+ "openviking.parse.accessors.git_accessor.os.killpg"
+ ) as killpg:
+ with pytest.raises(asyncio.CancelledError):
+ await accessor._run_git(
+ ["git", "clone", "https://git.example.com/org/private.git"]
+ )
+
+ killpg.assert_called_once_with(43210, signal.SIGKILL)
+ process.kill.assert_not_called()
+ process.wait.assert_awaited_once_with()
+
+ async def test_cancelled_git_process_falls_back_to_direct_kill_without_pgid(
+ self,
+ accessor: GitAccessor,
+ ) -> None:
+ process = SimpleNamespace(
+ pid=43210,
+ communicate=AsyncMock(side_effect=asyncio.CancelledError),
+ kill=Mock(),
+ wait=AsyncMock(),
+ )
+ with patch(
+ "openviking.parse.accessors.git_accessor.asyncio.create_subprocess_exec",
+ new=AsyncMock(return_value=process),
+ ):
+ with patch(
+ "openviking.parse.accessors.git_accessor.os.name", "nt"
+ ):
+ with pytest.raises(asyncio.CancelledError):
+ await accessor._run_git(
+ ["git", "clone", "https://git.example.com/org/private.git"]
+ )
process.kill.assert_called_once_with()
process.wait.assert_awaited_once_with()
+ async def test_cancelled_git_reap_is_bounded_when_wait_hangs(
+ self,
+ accessor: GitAccessor,
+ ) -> None:
+ async def hang_forever() -> None:
+ await asyncio.Event().wait()
+
+ process = SimpleNamespace(
+ pid=43210,
+ communicate=AsyncMock(side_effect=asyncio.CancelledError),
+ kill=Mock(),
+ wait=AsyncMock(side_effect=hang_forever),
+ )
+ accessor.CANCEL_REAP_TIMEOUT = 0.01
+ with patch(
+ "openviking.parse.accessors.git_accessor.asyncio.create_subprocess_exec",
+ new=AsyncMock(return_value=process),
+ ):
+ with patch(
+ "openviking.parse.accessors.git_accessor.os.killpg"
+ ) as killpg:
+ loop = asyncio.get_running_loop()
+ start = loop.time()
+ with pytest.raises(asyncio.CancelledError):
+ await accessor._run_git(
+ ["git", "clone", "https://git.example.com/org/private.git"]
+ )
+ elapsed = loop.time() - start
+
+ killpg.assert_called_once_with(43210, signal.SIGKILL)
+ assert elapsed < 1.0
+
@pytest.mark.parametrize(
("source", "repo_name"),
[