Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
Resource Service for OpenViking.
Provides resource management operations: add_resource, add_skill, wait_processed.
"""
import asyncio
import contextlib
import inspect
import json
import os
import signal
import time
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field, replace
from pathlib import Path
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from urllib.parse import urlparse
from uuid import uuid4
from openviking.connector.auth import (
OAUTH_REF_ARG,
feishu_auth_scope,
is_external_feishu_auth,
validate_feishu_auth_args,
)
from openviking.core.namespace import is_content_root_uri
from openviking.observability.http_error_context import sanitize_public_http_error
from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.mode import ParseMode, normalize_parse_mode
from openviking.parse.network import (
ParserNetworkContext,
ParserNetworkOperation,
overlay_parser_subprocess_env,
)
from openviking.parse.parsers.constants import MPEG_TS_EXTENSION_ALIAS
from openviking.resource.feishu_watch_auth import (
FEISHU_ACCESS_TOKEN_ARG,
FEISHU_APP_ID_ARG,
FEISHU_APP_SECRET_ARG,
FEISHU_AUTH_PROVIDER,
FEISHU_REFRESH_TOKEN_ARG,
FeishuAppCredentials,
create_feishu_auth_state,
is_feishu_auth_state,
load_feishu_app_credentials,
)
from openviking.resource.git_watch_auth import (
create_git_http_auth_state,
git_http_auth_config_from_state,
is_git_http_auth_state,
)
from openviking.resource.processing_mode import (
DEFAULT_PROCESSING_MODE,
ProcessingMode,
normalize_processing_mode,
)
from openviking.server.identity import RequestContext
from openviking.server.local_input_guard import (
is_remote_resource_source,
require_remote_resource_source,
)
from openviking.server.user_config import (
effective_resource_add_target,
effective_skill_add_target,
)
from openviking.service.task_domain import TaskPatch
from openviking.storage.acl import AclAction, AclSpec
from openviking.storage.pathlock_transfer import PathLockGuard
from openviking.storage.queuefs import QueueManager, get_queue_manager
from openviking.storage.queuefs.add_resource_msg import AddResourcePhase
from openviking.storage.viking_fs import LS_ALL_NODES, VikingFS
from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.telemetry import get_current_telemetry, register_telemetry, unregister_telemetry
from openviking.utils import is_git_repo_url, is_github_url, parse_code_hosting_url
from openviking.utils.git_auth import (
GitHttpAuthConfig,
build_git_http_auth_env,
is_git_https_url,
parse_git_http_auth_config,
raise_git_auth_error,
reject_git_http_userinfo,
)
from openviking.utils.ingest_options import IngestOptions
from openviking.utils.media_processor import _smart_stem
from openviking.utils.network_guard import ensure_public_remote_target
from openviking.utils.path_safety import normalize_storage_target_uri
from openviking.utils.resource_processor import ResourceProcessor
from openviking.utils.skill_processor import SkillProcessingPreparation, SkillProcessor
from openviking_cli.exceptions import (
ConflictError,
DeadlineExceededError,
FailedPreconditionError,
InternalError,
InvalidArgumentError,
NotInitializedError,
OpenVikingError,
)
from openviking_cli.utils import get_logger
if TYPE_CHECKING:
from openviking.connector.delegate import ConnectorDelegate
from openviking.parse.accessors.base import LocalResource
from openviking.resource.shared_source import SharedSource
from openviking.resource.staged_source import StagedSource
from openviking.resource.watch_manager import WatchManager, WatchTask
from openviking.resource.watch_scheduler import WatchScheduler
from openviking.service.resource_memory_link_service import ResourceMemoryLinkService
logger = get_logger(__name__)
_ADD_RESOURCE_ARGS_RESERVED_FIELDS = frozenset(
{
"path",
"ctx",
"to",
"to_is_directory",
"parent",
"reason",
"instruction",
"wait",
"timeout",
"build_index",
"summarize",
"processing_mode",
"watch_interval",
"is_active",
"skip_watch_management",
"allow_local_path_resolution",
"enforce_public_remote_targets",
"resource_lock",
"stage_callback",
"args",
"strict",
"source_name",
"ignore_dirs",
"include",
"exclude",
"directly_upload_media",
"preserve_structure",
"create_parent",
"telemetry",
"request_validator",
"understanding_response_id",
"understanding_file_id",
"parser_backend",
"resolved_extension",
"defer_post_processing",
"prepared_resource",
"tags",
"tag_mode",
"acl",
"internal_task",
"target_validator",
}
)
_ADD_RESOURCE_TRANSIENT_ARGS = frozenset({"tos_signature", "tos_access"})
_ADD_RESOURCE_TAG_MODES = frozenset({"replace", "append", "clear"})
_DISPLAY_RESOURCE_CONFIG_FIELDS = frozenset({"resource_name", "resource_tag"})
_INTERNAL_INGESTION_FIELDS = frozenset(
{
"manage_watch",
"parser_args",
"resource_lock",
"route_source",
"skip_watch_management",
"stage_callback",
"to_is_directory",
"watch_auth_state",
"understanding_response_id",
"understanding_file_id",
"parser_backend",
"resolved_extension",
"prepared_resource",
"target_validator",
}
)
@dataclass
class _ResourceSourceInfo:
source_name: Optional[str] = None
source_path: Optional[str] = None
source_format: Optional[str] = None
head_sha: Optional[str] = None
@dataclass
class _NormalizedAddResourceArgs:
processor_kwargs: Dict[str, Any]
watch_auth_state: Optional[Dict[str, Any]] = None
parse_mode: ParseMode = ParseMode.DEFAULT
@dataclass
class _SourcePlan:
path: str
source_identity: _ResourceSourceInfo
processor_args: Dict[str, Any]
task_auth: Dict[str, Any] = field(repr=False)
staged_source: Optional["StagedSource"] = None
shared_source: Optional["SharedSource"] = None
understanding_response_id: Optional[str] = None
understanding_file_id: Optional[str] = None
defer_unnamed_target: bool = False
def _normalized_resource_config(value: Any) -> Dict[str, str]:
if not isinstance(value, dict):
return {}
result: Dict[str, str] = {}
for key in _DISPLAY_RESOURCE_CONFIG_FIELDS:
item = value.get(key)
if isinstance(item, str) and item.strip():
result[key] = item.strip()
return result
class ResourceService:
"""Resource management service."""
def __init__(
self,
vikingdb: Optional[VikingDBManager] = None,
viking_fs: Optional[VikingFS] = None,
resource_processor: Optional[ResourceProcessor] = None,
skill_processor: Optional[SkillProcessor] = None,
watch_scheduler: Optional["WatchScheduler"] = None,
resource_memory_link_service: Optional["ResourceMemoryLinkService"] = None,
runtime_config_manager: Optional[Any] = None,
):
self._vikingdb = vikingdb
self._viking_fs = viking_fs
self._resource_processor = resource_processor
self._skill_processor = skill_processor
self._watch_scheduler = watch_scheduler
self._resource_memory_link_service = resource_memory_link_service
self._runtime_config_manager = runtime_config_manager
self._background_tasks: set[asyncio.Task[Any]] = set()
self._connector_delegate: Optional["ConnectorDelegate"] = None
self.compile_versions: Optional[Any] = None
def set_dependencies(
self,
vikingdb: VikingDBManager,
viking_fs: VikingFS,
resource_processor: ResourceProcessor,
skill_processor: SkillProcessor,
watch_scheduler: Optional["WatchScheduler"] = None,
resource_memory_link_service: Optional["ResourceMemoryLinkService"] = None,
runtime_config_manager: Optional[Any] = None,
) -> None:
"""Set dependencies (for deferred initialization)."""
self._vikingdb = vikingdb
self._viking_fs = viking_fs
self._resource_processor = resource_processor
self._skill_processor = skill_processor
self._watch_scheduler = watch_scheduler
self._resource_memory_link_service = resource_memory_link_service
self._runtime_config_manager = runtime_config_manager
def _get_watch_manager(self) -> Optional["WatchManager"]:
if not self._watch_scheduler:
return None
return self._watch_scheduler.watch_manager
async def _ensure_resource_target_available(
self, uri: str, ctx: RequestContext, *, is_parent: bool = False
) -> None:
"""Reject active Compile output destinations in this account before ingestion.
A parent only supplies a new child's location; an exact destination mirrors
its subtree and must also exclude Compile targets nested beneath it.
"""
if self.compile_versions is None or not uri:
return
uri = normalize_storage_target_uri(uri)
await self._viking_fs._ensure_access(uri, ctx, action=AclAction.WRITE)
target = await self.compile_versions.owning_target(ctx.account_id, uri)
if target is None and not is_parent:
targets = await self.compile_versions._targets_under(ctx.account_id, uri)
if targets:
target = targets[0]
if target is not None:
raise ConflictError(
"Add-resource destination overlaps a Compile target.",
resource="COMPILE_TARGET",
)
async def _hold_watch_execution(self, task_id: str) -> bool:
"""Keep the scheduler from running *task_id* while its first round is in flight."""
hold = getattr(self._watch_scheduler, "hold_execution", None)
if hold is None:
return True
return bool(await hold(task_id))
async def _release_watch_execution(self, task_id: str) -> None:
release = getattr(self._watch_scheduler, "release_execution", None)
if release is not None:
await release(task_id)
async def record_watch_execution(
self,
task_id: str,
*,
status: str,
execution_task_id: Optional[str] = None,
error: Optional[str] = None,
source_checkpoint: Optional[Dict[str, Any]] = None,
) -> None:
"""Record a pre-created Watch's first round and release its scheduler hold."""
sanitized_error = (
sanitize_public_http_error(code="WATCH_EXECUTION_FAILED", message=error).message
if error
else None
)
try:
watch_manager = self._get_watch_manager()
if watch_manager is not None:
await watch_manager.record_execution(
task_id,
status=status,
execution_task_id=execution_task_id,
error=error,
)
if status == "completed" and source_checkpoint is not None:
await watch_manager.update_source_checkpoint(
task_id,
source_checkpoint,
)
logger.info(
"[ResourceService] Watch first round recorded: watch_task_id=%s status=%s "
"execution_task_id=%s%s",
task_id,
status,
execution_task_id,
f" error={sanitized_error}" if sanitized_error else "",
)
finally:
await self._release_watch_execution(task_id)
def _sanitize_watch_processor_kwargs(self, processor_kwargs: Dict[str, Any]) -> Dict[str, Any]:
sanitized: Dict[str, Any] = {}
for key, value in processor_kwargs.items():
if key in {
"auth_config",
"lark_file",
FEISHU_ACCESS_TOKEN_ARG,
FEISHU_REFRESH_TOKEN_ARG,
OAUTH_REF_ARG,
"parser_backend",
"resolved_extension",
"understanding_response_id",
"understanding_file_id",
"temp_file_id",
"git_head_sha_hint",
}:
continue
try:
json.dumps(value, ensure_ascii=False)
except TypeError:
continue
sanitized[key] = value
return sanitized
def _watch_processor_kwargs(
self,
processor_kwargs: Dict[str, Any],
tags: Optional[List[str]],
tag_mode: str,
) -> Dict[str, Any]:
watch_kwargs = self._sanitize_watch_processor_kwargs(processor_kwargs)
if tags is not None or tag_mode == "clear":
if tags is not None:
watch_kwargs["tags"] = tags
watch_kwargs["tag_mode"] = tag_mode
return watch_kwargs
@staticmethod
def _infer_watch_source_type(path: str) -> Optional[str]:
if not path:
return None
from openviking.parse.accessors.feishu_accessor import FeishuAccessor
if FeishuAccessor._is_feishu_url(path):
return "feishu"
if is_git_repo_url(path):
return "git"
if is_remote_resource_source(path):
return "url"
return "local"
def _validate_add_resource_tag_policy(
self,
*,
tags: Optional[List[str]],
tag_mode: str,
) -> None:
if (tags is not None or tag_mode == "clear") and tag_mode not in _ADD_RESOURCE_TAG_MODES:
raise InvalidArgumentError(f"unsupported tag mode: {tag_mode}")
@staticmethod
def _ensure_single_resource_target(
*,
to: Optional[str],
parent: Optional[str],
) -> None:
if (to or "").strip() and (parent or "").strip():
raise InvalidArgumentError("Cannot specify both 'to' and 'parent' at the same time.")
@staticmethod
def _initial_source_checkpoint(
meta: Any,
*,
instruction: str = "",
) -> Optional[Dict[str, Any]]:
"""Build the watch source checkpoint from the imported revision meta.
The accessor records ``resolved_head_sha`` for the exact imported
revision: a resolved branch tip or the supplied commit-only pin. The
tracked branch or commit selection itself lives in processor_kwargs.
The instruction is included so changing it forces one refresh even
when the source revision is unchanged. Failed branch probes leave the
checkpoint unset (fail open).
"""
head_sha = meta.get("resolved_head_sha") if isinstance(meta, dict) else None
if head_sha:
return {
"kind": "git",
"head_sha": head_sha,
"instruction": instruction,
}
return None
async def _manage_watch_if_needed(
self,
*,
watch_manager: Optional["WatchManager"],
manage_watch: bool,
watch_interval: float,
to: str,
parent: str,
to_is_directory: bool,
root_uri: str,
path: str,
reason: str,
instruction: str,
build_index: bool,
summarize: bool,
processing_mode: ProcessingMode,
processor_kwargs: Dict[str, Any],
watch_auth_state: Optional[Dict[str, Any]],
ctx: RequestContext,
source_type: Optional[str] = None,
connector_states: Optional[Dict[str, Any]] = None,
is_active: Optional[bool] = None,
on_watch_ready: Optional[Callable[[str], None]] = None,
) -> None:
if not watch_manager or not manage_watch:
return
telemetry = get_current_telemetry()
with telemetry.measure("resource.watch"):
if watch_interval > 0:
watch_to = to
parent_uri = parent
if not watch_to:
watch_to = root_uri
parent_uri = None
if not watch_to:
raise InvalidArgumentError(
"watch_interval > 0 requires a stable target URI. "
"Pass 'to' explicitly, or add a resource type that returns root_uri."
)
if processor_kwargs.get("temp_file_id"):
# An uploaded source is a static snapshot, so a watch task recorded
# against it would re-process the frozen snapshot every interval.
raise InvalidArgumentError(
"watch_interval > 0 is not supported for uploaded content: an "
"upload is a static snapshot, so the watch would re-process "
"stale content forever. Watch a URL / "
"sitemap / RSS source instead, or re-add the resource when the "
"source changes."
)
try:
sanitized = self._sanitize_watch_processor_kwargs(processor_kwargs)
watch = await self._handle_watch_task_creation(
path=path,
to_uri=watch_to,
to_is_directory=to_is_directory,
parent_uri=parent_uri,
reason=reason,
instruction=instruction,
watch_interval=watch_interval,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
processor_kwargs=sanitized,
auth_state=watch_auth_state,
connector_states=connector_states,
source_type=source_type or self._infer_watch_source_type(path),
ctx=ctx,
is_active=is_active is not False,
)
if watch is not None and on_watch_ready is not None:
on_watch_ready(watch.task_id)
except ConflictError:
raise
except Exception as e:
logger.warning(
f"[ResourceService] Failed to create watch task for {watch_to}: {e}"
)
elif to:
try:
await self._handle_watch_task_cancellation(to_uri=to, ctx=ctx)
except ConflictError:
raise
except Exception as e:
logger.warning(f"[ResourceService] Failed to cancel watch task for {to}: {e}")
async def _normalize_add_resource_args(
self,
args: Optional[Dict[str, Any]],
*,
watch_interval: float,
ctx: RequestContext,
allowed_reserved_fields: Optional[set[str]] = None,
) -> _NormalizedAddResourceArgs:
if args is None:
return _NormalizedAddResourceArgs({})
if not isinstance(args, dict):
raise InvalidArgumentError("args must be an object.")
if not args:
return _NormalizedAddResourceArgs({})
reserved_fields = _ADD_RESOURCE_ARGS_RESERVED_FIELDS - (allowed_reserved_fields or set())
reserved = sorted(set(args).intersection(reserved_fields))
if reserved:
raise InvalidArgumentError(
"args cannot contain core add_resource fields: " + ", ".join(reserved)
)
normalized = dict(args)
validate_feishu_auth_args(normalized)
raw_parse_mode = normalized.pop("parse_mode", ParseMode.DEFAULT)
try:
parse_mode = normalize_parse_mode(raw_parse_mode)
except InvalidArgumentError as exc:
raise InvalidArgumentError(str(exc).replace("parse_mode", "args.parse_mode")) from exc
token = normalized.get(FEISHU_ACCESS_TOKEN_ARG)
refresh_token = normalized.pop(FEISHU_REFRESH_TOKEN_ARG, None)
app_id = normalized.pop(FEISHU_APP_ID_ARG, None)
app_secret = normalized.pop(FEISHU_APP_SECRET_ARG, None)
has_app_credentials = app_id is not None or app_secret is not None
watch_auth_state = None
if token is not None:
if not isinstance(token, str) or not token.strip():
raise InvalidArgumentError("args.feishu_access_token must be a non-empty string.")
token = token.strip()
normalized[FEISHU_ACCESS_TOKEN_ARG] = token
if watch_interval > 0:
if not isinstance(refresh_token, str) or not refresh_token.strip():
raise InvalidArgumentError(
"args.feishu_refresh_token must be a non-empty string when "
"args.feishu_access_token is used with watch_interval > 0."
)
if self._runtime_config_manager is None:
raise RuntimeError("Runtime config manager is not initialized")
from openviking.config.feishu import get_effective_feishu_config
feishu_config = await get_effective_feishu_config(
self._runtime_config_manager,
ctx.account_id,
)
app_credentials = self._load_feishu_credentials_for_watch(
app_id, app_secret, feishu_config
)
watch_auth_state = create_feishu_auth_state(
token,
refresh_token.strip(),
app_credentials,
persist_app_secret=app_id is not None or app_secret is not None,
)
elif refresh_token is not None:
raise InvalidArgumentError(
"args.feishu_refresh_token is only supported with "
"args.feishu_access_token and watch_interval > 0."
)
elif has_app_credentials:
raise InvalidArgumentError(
"args.feishu_app_id and args.feishu_app_secret are only supported with "
"args.feishu_access_token and watch_interval > 0."
)
elif refresh_token is not None:
raise InvalidArgumentError(
"args.feishu_refresh_token requires args.feishu_access_token."
)
elif has_app_credentials:
raise InvalidArgumentError(
"args.feishu_app_id and args.feishu_app_secret require "
"args.feishu_access_token and watch_interval > 0."
)
return _NormalizedAddResourceArgs(normalized, watch_auth_state, parse_mode)
def _load_feishu_credentials_for_watch(
self,
app_id: Any,
app_secret: Any,
config,
) -> FeishuAppCredentials:
supplied = app_id is not None or app_secret is not None
if supplied and (
not isinstance(app_id, str)
or not app_id.strip()
or not isinstance(app_secret, str)
or not app_secret.strip()
):
raise InvalidArgumentError(
"args.feishu_app_id and args.feishu_app_secret must be non-empty strings "
"and provided together."
)
try:
credentials = load_feishu_app_credentials(
config=config,
app_id=app_id.strip() if supplied else None,
app_secret=app_secret.strip() if supplied else None,
)
except Exception as exc:
raise InvalidArgumentError(
"Feishu user-token watch requires FEISHU_APP_ID and "
"FEISHU_APP_SECRET, or feishu.app_id and feishu.app_secret in ov.conf."
) from exc
# A user refresh token is bound to the Feishu application that issued
# it. Persist the effective app identity even when the caller used the
# account default, so a later account-config change cannot pair an old
# refresh token with a different app.
return credentials
def _ensure_initialized(self) -> None:
"""Ensure all dependencies are initialized."""
if not self._resource_processor:
raise NotInitializedError("ResourceProcessor")
if not self._skill_processor:
raise NotInitializedError("SkillProcessor")
if not self._viking_fs:
raise NotInitializedError("VikingFS")
async def _lock_to_handoff_payload(self, lock_ref: Any) -> Optional[Dict[str, Any]]:
"""Convert either a native pathlock ref or legacy lease into a handoff payload."""
if lock_ref is None:
return None
async_agfs = getattr(self._viking_fs, "_async_agfs", None)
if async_agfs is not None:
return await PathLockGuard(async_agfs, lock_ref).to_handoff()
to_handoff = getattr(lock_ref, "to_handoff", None)
if callable(to_handoff):
handoff = to_handoff()
return handoff.to_dict() if hasattr(handoff, "to_dict") else handoff
return lock_ref if isinstance(lock_ref, dict) else None
async def _handoff_lock_ref(self, lock_ref: Any) -> None:
"""Transfer ownership for either a native pathlock ref or legacy lease."""
if lock_ref is None:
return
async_agfs = getattr(self._viking_fs, "_async_agfs", None)
if async_agfs is not None:
await async_agfs.pathlock_handoff(lock_ref)
return
handoff = getattr(lock_ref, "handoff", None)
if callable(handoff):
result = handoff()
if inspect.isawaitable(result):
await result
async def _adopt_lock_ref(self, handoff_ref: Any) -> Any:
"""Recover ownership from a handoff after enqueue fails."""
async_agfs = getattr(self._viking_fs, "_async_agfs", None)
if async_agfs is not None:
return await async_agfs.pathlock_adopt(handoff_ref)
adopt = getattr(handoff_ref, "adopt", None)
if callable(adopt):
result = adopt()
return await result if inspect.isawaitable(result) else result
raise RuntimeError("Lock handoff cannot be adopted")
async def _release_lock_ref(self, lock_ref: Any) -> None:
"""Release either a native pathlock ref or legacy lease."""
if lock_ref is None:
return
async_agfs = getattr(self._viking_fs, "_async_agfs", None)
if async_agfs is not None:
await async_agfs.pathlock_release(lock_ref)
return
close = getattr(lock_ref, "close", None)
if callable(close):
result = close()
if inspect.isawaitable(result):
await result
async def _cleanup_reserved_target_if_empty(
self,
*,
root_uri: str,
ctx: RequestContext,
resource_lock: Dict[str, Any],
) -> bool:
"""Remove a newly reserved target only while it is still empty."""
ctx = replace(ctx, bypass_acl=True)
try:
if not await self._viking_fs.exists(root_uri, ctx=ctx):
return True
stat = await self._viking_fs.stat(root_uri, ctx=ctx, skip_count=True)
if not isinstance(stat, dict) or not stat.get("isDir"):
return False
entries = await self._viking_fs.ls(
root_uri,
show_all_hidden=True,
node_limit=LS_ALL_NODES,
ctx=ctx,
)
if any(entry.get("name") not in {None, "", ".", ".."} for entry in entries):
return False
await self._viking_fs.rm(
root_uri,
recursive=True,
ctx=ctx,
lease_ref=resource_lock,
)
return True
except Exception as exc:
logger.warning(
"[ResourceService] Failed to clean empty reserved target %s: %s",
root_uri,
exc,
)
return False
async def close_background_tasks(self) -> None:
"""Cancel in-flight connector monitoring tasks during service shutdown."""
if not self._background_tasks:
return
tasks = list(self._background_tasks)
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
self._background_tasks.clear()
def _unregister_telemetry_after_task(
self,
task_id: str,
telemetry_id: str,
*,
account_id: str,
user_id: str,
) -> None:
async def wait_and_unregister() -> None:
from openviking.service.task_tracker import get_task_tracker
try:
await get_task_tracker().wait(
task_id,
account_id=account_id,
user_id=user_id,
)
finally:
unregister_telemetry(telemetry_id)
background = asyncio.create_task(wait_and_unregister())
self._background_tasks.add(background)
background.add_done_callback(self._background_tasks.discard)
async def _enqueue_add_resource_job(
self,
msg: Any,
*,
queue_name: str,
resource_lock: Optional[Dict[str, Any]] = None,
task_auth: Optional[Dict[str, Any]] = None,
on_enqueued: Optional[Callable[[], None]] = None,
on_failed_with_lock: Optional[Callable[[Dict[str, Any]], Awaitable[None]]] = None,
) -> Any:
"""Persist a job and fully own the passed lock until handoff or release completes."""
from openviking.service.task_tracker import get_task_tracker
from openviking.service.task_work_index import bind_task_context
from openviking.storage.queuefs import get_queue_manager
tracker = get_task_tracker()
task = None
enqueued = False
lock_ops = getattr(self._viking_fs, "_async_agfs", None) or SimpleNamespace(
pathlock_handoff=self._handoff_lock_ref,
pathlock_adopt=self._adopt_lock_ref,
pathlock_release=self._release_lock_ref,
)
lock_guard = PathLockGuard(lock_ops, resource_lock)
try:
task = await tracker.create(
"add_resource",
resource_id=None if msg.defer_target_resolution else msg.root_uri,
account_id=msg.account_id,
user_id=msg.user_id,
task_id=msg.task_id,
meta=(
{"internal": True} if msg.internal_task else {"source_path": msg.source_path}
),
auth=task_auth,
)
task_account_id = getattr(task, "account_id", msg.account_id)
task_user_id = getattr(task, "user_id", msg.user_id)
async def dispatch() -> str:
nonlocal enqueued
with bind_task_context(task.task_id, task_account_id, task_user_id):
result = await get_queue_manager().enqueue(queue_name, msg.to_dict())
enqueued = True
if on_enqueued is not None:
on_enqueued()
return result
if resource_lock is not None:
await lock_guard.handoff_and_dispatch(msg.lock_handoff, dispatch)
else:
await dispatch()
await tracker.patch_task(
task.task_id,
TaskPatch(stage="queued"),
account_id=msg.account_id,
user_id=msg.user_id,
)
except BaseException:
if lock_guard.lease is not None:
try:
if on_failed_with_lock is not None:
await on_failed_with_lock(lock_guard.lease)
finally:
await lock_guard.close()
if task is not None and not enqueued:
await tracker.fail(
task.task_id,
"Failed to enqueue resource processing",
account_id=msg.account_id,
user_id=msg.user_id,
)
raise
return task
async def execute_add_resource_job(
self,
msg: Any,
*,
ctx: RequestContext,
resource_lock: Optional[Dict[str, Any]],
stage_callback: Callable[[str], Any],
task_auth: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Execute one durable add-resource job inside its QueueFS consumer."""
if msg.job_phase is AddResourcePhase.SOURCE:
from openviking.resource.staged_source import StagedSource, materialize_source
target_uri = msg.root_uri
parent_uri = None
queued_args = dict(msg.args)
legacy_backend = normalize_parser_backend(queued_args.pop("parser_backend", None))
feishu_prepared = queued_args.pop("_feishu_prepared_source", False)
parser_backend = legacy_backend or (
ParserBackend.UNDERSTANDING
if msg.understanding_response_id is not None
or msg.understanding_file_id is not None
else None
if feishu_prepared
else ParserBackend.INTERNAL
)
internal_kwargs: Dict[str, Any] = {"parser_backend": parser_backend}
if "resolved_extension" in queued_args:
internal_kwargs["resolved_extension"] = queued_args.pop("resolved_extension")
normalized_args = await self._normalize_add_resource_args(
queued_args,
ctx=ctx,
watch_interval=msg.watch_interval,
)
internal_kwargs.update(normalized_args.processor_kwargs)
if msg.strict:
internal_kwargs["strict"] = True
if msg.ignore_dirs is not None:
internal_kwargs["ignore_dirs"] = msg.ignore_dirs
if msg.include is not None:
internal_kwargs["include"] = msg.include
if msg.exclude is not None:
internal_kwargs["exclude"] = msg.exclude
if not msg.directly_upload_media:
internal_kwargs["directly_upload_media"] = False
if msg.preserve_structure is not None:
internal_kwargs["preserve_structure"] = msg.preserve_structure
if msg.create_parent:
internal_kwargs["create_parent"] = True
if msg.source_name is not None:
internal_kwargs["source_name"] = msg.source_name
if msg.git_head_sha_hint:
# One-shot SHA resolved by the add_resource preflight; lets
# the accessor download an immutable revision without a second
# ls-remote. Transient: never persisted into watch kwargs.
internal_kwargs["git_head_sha_hint"] = msg.git_head_sha_hint
if feishu_prepared:
from openviking.service.task_tracker import get_task_tracker
tracker = get_task_tracker()
task = await tracker.get(msg.task_id, ctx.account_id, ctx.user.user_id)
saved = dict(task.meta.get("feishu_responses", {})) if task else {}
async def save_response(entry: str, response_id: str) -> None:
await tracker.patch_task(
msg.task_id,
TaskPatch(meta={"feishu_responses": {entry: response_id}}),
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
saved[entry] = response_id
internal_kwargs["_feishu_checkpoint"] = (saved, save_response)
prepared_resource = None
if msg.staged_source is not None:
with get_current_telemetry().measure("resource.source_prepare"):
prepared_resource = await materialize_source(
StagedSource.from_dict(msg.staged_source),
viking_fs=self._viking_fs,
ctx=ctx,
)
elif msg.shared_source is not None:
from openviking.resource.shared_source import (
SharedSource,
materialize_shared_source,
)
with get_current_telemetry().measure("resource.source_prepare"):
prepared_resource = await materialize_shared_source(
SharedSource.from_dict(msg.shared_source),
viking_fs=self._viking_fs,
ctx=ctx,
)
if msg.defer_target_resolution:
from openviking_cli.utils.uri import VikingURI
target_uri = None
parent_uri = VikingURI(msg.root_uri).parent.uri
if msg.understanding_response_id is not None:
from openviking.parse.understanding_api import PREPARED_RESPONSE_ID_ARG
internal_kwargs[PREPARED_RESPONSE_ID_ARG] = msg.understanding_response_id
if msg.understanding_file_id is not None:
from openviking.parse.understanding_api import PREPARED_FILE_ID_ARG
internal_kwargs[PREPARED_FILE_ID_ARG] = msg.understanding_file_id
try:
async with feishu_auth_scope(
self._connector,
path=msg.path,
ctx=ctx,
args=internal_kwargs,
state=task_auth,
prepared=(
msg.understanding_response_id is not None
or msg.understanding_file_id is not None
),
) as auth_state:
auth_kwargs, watch_auth_state = self._restore_source_task_auth(
msg, auth_state or {}
)
internal_kwargs.update(auth_kwargs)
result = await self._execute_resource_ingestion(
path=msg.path,
ctx=ctx,
to=target_uri,
parent=parent_uri,
to_is_directory=msg.to_is_directory,
reason=msg.reason,
instruction=msg.instruction,
defer_post_processing=False,
build_index=msg.build_index,
summarize=msg.summarize,
processing_mode=msg.processing_mode,
parse_mode=msg.parse_mode,
watch_interval=msg.watch_interval,
is_active=msg.is_active,
manage_watch=not msg.skip_watch_management,
acl=msg.acl,
tags=msg.tags,
tag_mode=msg.tag_mode,
allow_local_path_resolution=msg.allow_local_path_resolution,
enforce_public_remote_targets=msg.enforce_public_remote_targets,
resource_lock=resource_lock,
stage_callback=stage_callback,
watch_auth_state=watch_auth_state,
prepared_resource=prepared_resource,
internal_task=msg.internal_task,
on_watch_ready=lambda task_id: setattr(msg, "watch_task_id", task_id),
**internal_kwargs,
)
except BaseException:
if msg.cleanup_empty_target_on_failure and resource_lock is not None:
await self._cleanup_reserved_target_if_empty(
root_uri=msg.root_uri,
ctx=ctx,
resource_lock=resource_lock,
)
raise
if (
result.get("status") == "error"
and msg.cleanup_empty_target_on_failure
and resource_lock is not None
):
await self._cleanup_reserved_target_if_empty(
root_uri=msg.root_uri,
ctx=ctx,
resource_lock=resource_lock,
)
if msg.staged_source is not None or msg.shared_source is not None:
result["source_path"] = msg.source_path
stage_result = stage_callback("processing_queue")
if inspect.isawaitable(stage_result):
await stage_result
return result
stage_result = stage_callback("processing_queue")
if inspect.isawaitable(stage_result):
await stage_result
return await self._resource_processor.finish_prepared_resource(
msg.prepared,
ctx=ctx,
resource_lock=resource_lock,
summarize=msg.summarize,
build_index=msg.build_index,
processing_mode=msg.processing_mode,
ingest_options=IngestOptions.from_search_tags(msg.tags, mode=msg.tag_mode),
)
def _restore_source_task_auth(
self,
msg: Any,
task_auth: Dict[str, Any],
) -> tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
"""Restore provider-specific request inputs from task-owned auth state."""
if not task_auth:
return {}, None
creating_watch = msg.watch_interval > 0 and not msg.skip_watch_management
if is_external_feishu_auth(task_auth):
return {}, task_auth if msg.watch_interval > 0 else None
if is_git_http_auth_state(task_auth):
auth_config = git_http_auth_config_from_state(task_auth, msg.path)
watch_auth_state = dict(task_auth) if creating_watch else None
return {"auth_config": auth_config}, watch_auth_state
if is_feishu_auth_state(task_auth):
token = task_auth.get("access_token")
if not isinstance(token, str) or not token.strip():
raise InvalidArgumentError("Stored Feishu task credentials are invalid.")
if creating_watch:
refresh_token = task_auth.get("refresh_token")
if not isinstance(refresh_token, str) or not refresh_token.strip():
raise InvalidArgumentError(
"Stored Feishu watch credentials are missing a refresh token."
)
watch_auth_state = dict(task_auth)
watch_auth_state.pop("domain", None)
watch_auth_state.pop("request_timeout", None)
else:
watch_auth_state = None
auth_kwargs = (
{FEISHU_ACCESS_TOKEN_ARG: token.strip()}
if msg.understanding_response_id is None and msg.understanding_file_id is None
else {}
)
return auth_kwargs, watch_auth_state
raise InvalidArgumentError("Unsupported task authentication provider.")
async def reacquire_add_resource_job_lock(
self,
root_uri: str,
ctx: RequestContext,
) -> Dict[str, Any]:
"""Acquire a fresh lock when a recovered job's old handoff was released."""
if not self._resource_processor or not self._viking_fs:
raise NotInitializedError("ResourceProcessor")
dst_path = self._viking_fs._uri_to_path(root_uri, ctx=ctx)
return await self._viking_fs._async_agfs.pathlock_acquire_tree(
dst_path,
timeout_secs=0.0,
)
async def _prepare_standard_source_plan(
self,
*,
path: str,
ctx: RequestContext,
mode: ParseMode,
allow_local_path_resolution: bool,
processor_kwargs: Dict[str, Any],
watch_auth_state: Optional[Dict[str, Any]],
shared_source: Optional["SharedSource"] = None,
) -> Optional[_SourcePlan]:
"""Freeze one durable standard-pipeline source before it crosses QueueFS."""
from openviking.parse.accessors.feishu_accessor import FeishuAccessor
from openviking.resource.staged_source import stage_source
source_name = processor_kwargs.get("source_name")
# A shared upload is already durable; reference it directly instead of
# downloading + re-staging a second copy. Its identity comes from the
# validated upload meta, so no accessor preflight is needed.
if shared_source is not None:
queued_args = {
key: value
for key, value in processor_kwargs.items()
if key not in _ADD_RESOURCE_ARGS_RESERVED_FIELDS | _ADD_RESOURCE_TRANSIENT_ARGS
}
queued_args = self._sanitize_watch_processor_kwargs(queued_args)
resolved_name = source_name or shared_source.original_filename or None
source_format = (
Path(shared_source.original_filename).suffix.lower().lstrip(".") or "file"
)
return _SourcePlan(
path=path,
source_identity=_ResourceSourceInfo(
source_name=resolved_name,
source_path=shared_source.original_filename or path,
source_format=source_format,
),
processor_args=queued_args,
task_auth={},
shared_source=shared_source,
)
git_source = is_git_repo_url(path)
feishu_source = FeishuAccessor._is_feishu_url(path)
remote_source = is_remote_resource_source(path)
local_source = False
if allow_local_path_resolution and len(path) <= 1024 and "\n" not in path:
with contextlib.suppress(OSError, ValueError):
local_source = Path(path).exists()
if not (git_source or feishu_source or remote_source or local_source):
return None
queued_args = {
key: value
for key, value in processor_kwargs.items()
if key not in _ADD_RESOURCE_ARGS_RESERVED_FIELDS | _ADD_RESOURCE_TRANSIENT_ARGS
}
queued_args = self._sanitize_watch_processor_kwargs(queued_args)
task_auth: Dict[str, Any] = {}
staged_source = None
understanding_response_id = None
understanding_file_id = None
defer_unnamed_target = False
if git_source:
reject_git_http_userinfo(path)
from openviking.connector.routing import credential_arg_names
credential_args = credential_arg_names("git", processor_kwargs)
if credential_args:
raise InvalidArgumentError("Native Git credentials must use args.auth_config.")
request_git_auth = parse_git_http_auth_config(processor_kwargs.get("auth_config"), path)
if request_git_auth is not None:
task_auth = create_git_http_auth_state(request_git_auth, path)
preflight_git_auth = request_git_auth
if preflight_git_auth is None and is_git_https_url(path) and is_github_url(path):
github_token = await self._resource_processor.github_token_for(path, ctx)
if github_token:
preflight_git_auth = GitHttpAuthConfig(
username="oauth2",
token=github_token,
)
source_info = await self._preflight_git_source(
path,
auth_config=preflight_git_auth,
branch=processor_kwargs.get("branch") or processor_kwargs.get("ref"),
commit=processor_kwargs.get("commit"),
)
source_name = source_name or source_info.source_name
source_info.source_name = source_name
elif feishu_source:
from openviking.parse.feishu_import import recursive_wiki
recursive = FeishuAccessor._parse_feishu_url(path)[0] == "wiki" and recursive_wiki(
processor_kwargs
)
token = processor_kwargs.get(FEISHU_ACCESS_TOKEN_ARG)
if isinstance(token, str) and token.strip():
task_auth = dict(
watch_auth_state
or {
"provider": FEISHU_AUTH_PROVIDER,
"access_token": token.strip(),
}
)
task_auth.pop("domain", None)
task_auth.pop("request_timeout", None)
if self._runtime_config_manager is None:
raise RuntimeError("Runtime config manager is not initialized")
from openviking.config.feishu import get_effective_feishu_config
feishu_config = await get_effective_feishu_config(
self._runtime_config_manager,
ctx.account_id,
)
feishu_kwargs = dict(processor_kwargs)
feishu_kwargs["feishu_config"] = feishu_config
preflight = await FeishuAccessor().preflight_source(
path,
feishu_access_token=token.strip() if isinstance(token, str) else None,
feishu_config=feishu_config,
**({"feishu_recursive": True} if recursive else {}),
)
source_name = source_name or preflight.source_name
source_info = _ResourceSourceInfo(
source_name=source_name,
source_path=path,
source_format=preflight.source_format,
)
defer_unnamed_target = True
direct_understanding = bool(
mode is ParseMode.DEFAULT
and self._resource_processor.should_use_understanding_directly(
path,
**feishu_kwargs,
)
)
if direct_understanding:
understanding_response_id = await self._resource_processor.submit_understanding(
path,
**feishu_kwargs,
)
if watch_auth_state is None:
task_auth = {}
else:
source_type, _ = FeishuAccessor._parse_feishu_url(path)
if source_type in {"folder", "file"} or recursive:
if processor_kwargs.get("lark_file") is not None:
raise InvalidArgumentError(
"Feishu sources requiring preparation use args.feishu_access_token "
"or configured Feishu application credentials; args.lark_file "
"is only supported for direct single-document Understanding imports."
)
# These sources choose a content backend after source preparation.
queued_args["parser_backend"] = processor_kwargs.get("parser_backend")
queued_args["_feishu_prepared_source"] = True
else:
prepared = await self._resource_processor.prepare_durable_source(
path,
ctx,
snapshot_required=local_source
or bool(
processor_kwargs.get("tos_signature") or processor_kwargs.get("tos_access")
),
parse_mode=mode,
allow_local_path_resolution=allow_local_path_resolution,
**processor_kwargs,
)
if prepared is None:
parsed_path = Path(urlparse(path).path)
source_name = source_name or parsed_path.name or None
source_info = _ResourceSourceInfo(
source_name=source_name,
source_path=path,
source_format=parsed_path.suffix.lower().lstrip(".") or "file",
)
else:
try:
resolved_extension, source_info = self._prepared_source_info(
prepared,
path,
source_name,
use_path_name=local_source,
)
if resolved_extension:
queued_args["resolved_extension"] = resolved_extension
if (
mode is ParseMode.DEFAULT
and self._resource_processor.should_use_understanding_api(prepared)
):
if processor_kwargs.get("temp_file_id"):
understanding_file_id = (
await self._resource_processor.upload_understanding_file(prepared)
)
else:
understanding_response_id = (
await self._resource_processor.submit_understanding(
prepared,
**processor_kwargs,
)
)
else:
staged_source = await stage_source(
prepared,
viking_fs=self._viking_fs,
ctx=ctx,
)
finally:
prepared.cleanup()
return _SourcePlan(
path=path,
source_identity=source_info,
processor_args=queued_args,
task_auth=task_auth,
staged_source=staged_source,
understanding_response_id=understanding_response_id,
understanding_file_id=understanding_file_id,
defer_unnamed_target=defer_unnamed_target,
)
@staticmethod
def _prepared_source_info(
resource: "LocalResource",
path: str,
source_name: Optional[str],
*,
use_path_name: bool = False,
) -> tuple[str, _ResourceSourceInfo]:
resolved_extension = str(
resource.meta.get("resolved_extension")
or resource.meta.get("extension")
or resource.path.suffix
or ""
)
resolved_name = (
source_name
or resource.meta.get("original_filename")
or resource.meta.get("resolved_name")
)
if not resolved_name and use_path_name:
resolved_name = resource.path.name
source_format = "directory" if resource.path.is_dir() else resolved_extension.lstrip(".")
if not source_format:
source_format = "file"
if resolved_extension.lower().lstrip(".") == MPEG_TS_EXTENSION_ALIAS:
source_format = "video"
return (
resolved_extension,
_ResourceSourceInfo(
source_name=resolved_name,
source_path=resource.original_source or path,
source_format=source_format,
),
)
async def _enqueue_source_plan(
self,
plan: _SourcePlan,
*,
ctx: RequestContext,
to: str,
parent: str,
create_parent: bool,
reason: str,
instruction: str,
build_index: bool,
summarize: bool,
processing_mode: ProcessingMode,
mode: ParseMode,
watch_interval: float,
manage_watch: bool,
tags: Optional[List[str]],
tag_mode: str,
acl: AclSpec | None,
to_is_directory: Optional[bool],
allow_local_path_resolution: bool,
enforce_public_remote_targets: bool,
processor_kwargs: Dict[str, Any],
internal_task: bool,
is_active: Optional[bool] = None,
) -> Dict[str, Any]:
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
planned_to_is_directory = to_is_directory if to_is_directory is not None else bool(to)
target_to_is_exact = bool(to and not is_content_root_uri(to, kind="resource"))
defer_candidate_resolution = bool(
(plan.defer_unnamed_target and plan.source_identity.source_name is None and not to)
or (mode is ParseMode.NO_SPLIT and not target_to_is_exact)
)
root_uri = ""
resource_lock = None
defer_target_resolution = False
staged_enqueued = False
try:
(
root_uri,
resource_lock,
defer_target_resolution,
cleanup_empty_target_on_failure,
) = await self._plan_source_job_target(
path=plan.path,
ctx=ctx,
to=to,
parent=parent,
create_parent=create_parent,
source_info=plan.source_identity,
defer_candidate_resolution=defer_candidate_resolution,
to_is_directory=planned_to_is_directory,
)
lock_handoff = await self._lock_to_handoff_payload(resource_lock)
message_path = plan.path
if processor_kwargs.get("temp_file_id") and plan.understanding_file_id is not None:
message_path = (
plan.source_identity.source_name
or plan.source_identity.source_path
or "uploaded-file"
)
msg = AddResourceMsg(
task_id=str(uuid4()),
job_phase=AddResourcePhase.SOURCE,
path=message_path,
source_path=(plan.source_identity.source_name or "")
if processor_kwargs.get("temp_file_id")
else plan.source_identity.source_path or plan.path,
root_uri=root_uri,
staged_source=(
plan.staged_source.to_dict() if plan.staged_source is not None else None
),
shared_source=(
plan.shared_source.to_dict() if plan.shared_source is not None else None
),
telemetry_id=get_current_telemetry().telemetry_id or None,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
group_ids=list(ctx.group_ids),
role=str(ctx.role),
actor_peer_id=ctx.actor_peer_id,
bypass_acl=ctx.bypass_acl,
lock_handoff=lock_handoff,
reason=reason,
instruction=instruction,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
parse_mode=mode.value,
watch_interval=watch_interval,
is_active=is_active,
acl=acl.model_dump(mode="json", exclude_none=True) if acl is not None else None,
skip_watch_management=not manage_watch,
tags=tags,
tag_mode=tag_mode,
allow_local_path_resolution=(
True
if (plan.staged_source is not None or plan.shared_source is not None)
else allow_local_path_resolution
),
enforce_public_remote_targets=(
enforce_public_remote_targets
and plan.staged_source is None
and plan.shared_source is None
),
strict=bool(processor_kwargs.get("strict", False)),
ignore_dirs=processor_kwargs.get("ignore_dirs"),
include=processor_kwargs.get("include"),
exclude=processor_kwargs.get("exclude"),
directly_upload_media=bool(processor_kwargs.get("directly_upload_media", True)),
preserve_structure=processor_kwargs.get("preserve_structure"),
create_parent=create_parent,
source_name=plan.source_identity.source_name,
to_is_directory=planned_to_is_directory,
args=plan.processor_args,
git_head_sha_hint=plan.source_identity.head_sha,
defer_target_resolution=defer_target_resolution,
cleanup_empty_target_on_failure=cleanup_empty_target_on_failure,
understanding_response_id=plan.understanding_response_id,
understanding_file_id=plan.understanding_file_id,
internal_task=internal_task,
)
def transfer_staged_source() -> None:
nonlocal staged_enqueued
staged_enqueued = True
queue_name = (
QueueManager.EXTERNAL_PARSE
if plan.understanding_response_id is not None
or plan.understanding_file_id is not None
else QueueManager.ADD_RESOURCE
)
enqueue_lock = resource_lock
resource_lock = None
async def cleanup_enqueue_failure(lock: Dict[str, Any]) -> None:
if cleanup_empty_target_on_failure:
await self._cleanup_reserved_target_if_empty(
root_uri=root_uri,
ctx=ctx,
resource_lock=lock,
)
task = await self._enqueue_add_resource_job(
msg,
queue_name=queue_name,
resource_lock=enqueue_lock,
task_auth=plan.task_auth,
on_enqueued=(transfer_staged_source if plan.staged_source is not None else None),
on_failed_with_lock=cleanup_enqueue_failure,
)
except BaseException:
if resource_lock is not None:
await self._release_lock_ref(resource_lock)
if plan.staged_source is not None and not staged_enqueued:
await self._viking_fs.delete_temp(plan.staged_source.temp_uri, ctx=ctx)
raise
response = {"status": "success", "task_id": task.task_id, "source_path": msg.source_path}
if not defer_target_resolution:
response["root_uri"] = root_uri
return response
async def _plan_source_job_target(
self,
*,
path: str,
ctx: RequestContext,
to: str,
parent: str,
create_parent: bool,
source_info: _ResourceSourceInfo,
defer_candidate_resolution: bool,
to_is_directory: bool,
) -> tuple[str, Optional[Dict[str, Any]], bool, bool]:
"""Resolve the target and track ownership of a newly reserved empty path."""
if not self._resource_processor or not self._viking_fs:
raise NotInitializedError("ResourceProcessor")
doc_name = self._target_doc_name(path, source_info)
source_path = source_info.source_path or source_info.source_name or path
root_uri, candidate_uri = await self._resource_processor.tree_builder.resolve_target_uri(
ctx=ctx,
doc_name=doc_name,
scope="resources",
to_uri=to,
parent_uri=parent,
source_path=source_path,
source_format=source_info.source_format,
create_parent=create_parent,
)
if candidate_uri and defer_candidate_resolution:
await self._ensure_resource_target_available(
candidate_uri.rpartition("/")[0], ctx, is_parent=True
)
await self._resource_processor.ensure_candidate_parent_write_access(
candidate_uri=candidate_uri,
ctx=ctx,
)
return root_uri, None, True, False
if candidate_uri:
root_uri, resource_lock = await self._resource_processor.reserve_unique_candidate(
candidate_uri=candidate_uri,
ctx=ctx,
target_validator=self._ensure_resource_target_available,
)
return root_uri, resource_lock, False, True
await self._ensure_resource_target_available(root_uri, ctx)
await self._viking_fs._ensure_access(root_uri, ctx, action=AclAction.WRITE)
# A tree lock may materialize an empty directory marker, so ownership
# must be recorded immediately before acquiring it. The zero-timeout
# lock still rejects compliant concurrent writers, and cleanup rechecks
# that the target is empty before deleting it.
target_preexisting = await self._viking_fs.exists(root_uri, ctx=ctx)
if to_is_directory and target_preexisting:
target_stat = await self._viking_fs.stat(root_uri, ctx=ctx, skip_count=True)
if not target_stat.get("isDir"):
raise FailedPreconditionError(
"Target URI already exists as a file and cannot be used as a "
f"resource directory: {root_uri}. Choose another URI, use "
"'parent' to add a new resource under a directory, or use "
"content/write to update the existing file.",
details={"resource": root_uri, "type": "file"},
)
dst_path = self._viking_fs._uri_to_path(root_uri, ctx=ctx)
resource_lock = await self._viking_fs._async_agfs.pathlock_acquire_tree(
dst_path,
timeout_secs=0.0,
)
try:
await self._viking_fs._ensure_access(root_uri, ctx, action=AclAction.WRITE)
except BaseException:
await self._release_lock_ref(resource_lock)
raise
return root_uri, resource_lock, False, not target_preexisting
@staticmethod
def _target_doc_name(
path: str,
source_info: _ResourceSourceInfo,
) -> str:
if source_info.source_name:
return _smart_stem(source_info.source_name)
if source_info.source_format == "repository":
parsed = parse_code_hosting_url(path)
if parsed:
return parsed.rsplit("/", 1)[-1]
return _smart_stem(Path(path).name or "resource")
async def _preflight_git_source(
self,
source: str,
*,
auth_config: Optional[GitHttpAuthConfig] = None,
branch: Optional[str] = None,
commit: Optional[str] = None,
) -> _ResourceSourceInfo:
# Normalize browser/tree URLs to the clone URL and resolve the tracked
# ref with the same logic the accessor/watch probe uses.
from openviking.parse.accessors.git_accessor import GitAccessor
repo_url, tracked_branch, tracked_commit = GitAccessor._resolve_probe_target(
source, branch=branch, commit=commit
) or (source, branch, commit)
# Target the exact tracked ref so the advertised SHA can be reused by
# the accessor as an immutable download hint. A pinned commit cannot be
# proven reachable by ls-remote, so HEAD only verifies repository
# access there; the fetch/checkout validates the exact SHA.
if tracked_commit:
ref = "HEAD"
ls_remote_args = ["git", "ls-remote", repo_url, "HEAD"]
elif tracked_branch:
ref = (
tracked_branch
if tracked_branch.startswith("refs/")
else f"refs/heads/{tracked_branch}"
)
ls_remote_args = ["git", "ls-remote", "--heads", repo_url, ref]
else:
ref = "HEAD"
ls_remote_args = ["git", "ls-remote", repo_url, "HEAD"]
proc = None
try:
env = build_git_http_auth_env(auth_config, repo_url) if auth_config is not None else None
env = overlay_parser_subprocess_env(
ParserNetworkContext(ParserNetworkOperation.GIT_COMMAND, repo_url),
env,
)
proc = await asyncio.create_subprocess_exec(
*ls_remote_args,
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
start_new_session=os.name == "posix",
)
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=10.0)
except BaseException as exc:
if proc is not None:
with contextlib.suppress(ProcessLookupError):
if os.name == "posix":
os.killpg(proc.pid, signal.SIGKILL)
else:
proc.kill()
with contextlib.suppress(Exception):
await asyncio.wait_for(asyncio.shield(proc.communicate()), timeout=1.0)
if isinstance(exc, asyncio.TimeoutError):
raise InvalidArgumentError(
"Cannot access Git repository; the preflight timed out after 10s."
) from exc
if isinstance(exc, Exception):
raise InvalidArgumentError(
"Cannot access Git repository during preflight."
) from exc
raise
if proc.returncode != 0:
raise_git_auth_error(stderr)
raise InvalidArgumentError("Cannot access Git repository; git ls-remote failed.")
# Best-effort SHA extraction only: an empty repository or a ref the
# server does not advertise still exits 0 and must pass preflight. In
# that case the hint stays unset and the clone path performs the
# definitive ref/error classification.
head_sha = None
if not tracked_commit:
head_sha = GitAccessor._parse_ls_remote_ref(
stdout.decode(errors="replace"), ref
)
repo_name = parse_code_hosting_url(source)
return _ResourceSourceInfo(
source_name=repo_name.rsplit("/", 1)[-1] if repo_name else None,
source_path=source,
source_format="repository",
head_sha=head_sha,
)
async def add_resource(
self,
path: str,
ctx: RequestContext,
to: Optional[str] = None,
parent: Optional[str] = None,
reason: str = "",
instruction: str = "",
wait: bool = False,
build_index: bool = True,
summarize: bool = False,
processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
watch_interval: float = 0,
is_active: Optional[bool] = None,
tags: Optional[List[str]] = None,
tag_mode: str = "replace",
allow_local_path_resolution: bool = True,
enforce_public_remote_targets: bool = False,
add_type: Optional[str] = None,
internal_task: bool = False,
args: Optional[Dict[str, Any]] = None,
connector_auth_config: Optional[Dict[str, Any]] = None,
shared_source: Optional["SharedSource"] = None,
acl: AclSpec | dict[str, Any] | None = None,
**kwargs,
) -> Dict[str, Any]:
"""Accept and route a new resource-add request."""
internal_fields = sorted(set(kwargs).intersection(_INTERNAL_INGESTION_FIELDS))
if internal_fields:
raise InvalidArgumentError(
"add_resource does not accept internal execution fields: "
+ ", ".join(internal_fields)
)
if isinstance(add_type, str):
add_type = add_type.strip() or None
return await self._submit_resource_ingestion(
path=path,
ctx=ctx,
add_type=add_type,
to=to,
parent=parent,
reason=reason,
instruction=instruction,
wait=wait,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
watch_interval=watch_interval,
is_active=is_active,
acl=acl,
manage_watch=True,
tags=tags,
tag_mode=tag_mode,
allow_local_path_resolution=allow_local_path_resolution,
enforce_public_remote_targets=enforce_public_remote_targets,
internal_task=internal_task,
args=args,
connector_auth_config=connector_auth_config,
shared_source=shared_source,
**kwargs,
)
async def refresh_resource(
self,
path: str,
ctx: RequestContext,
to: Optional[str] = None,
to_is_directory: Optional[bool] = None,
parent: Optional[str] = None,
reason: str = "",
instruction: str = "",
build_index: bool = True,
summarize: bool = False,
processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
watch_interval: float = 0,
allow_local_path_resolution: bool = True,
enforce_public_remote_targets: bool = False,
add_type: Optional[str] = None,
args: Optional[Dict[str, Any]] = None,
connector_states: Optional[Dict[str, Any]] = None,
connector_auth_config: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Dict[str, Any]:
"""Submit a scheduled refresh without changing its watch task."""
return await self._submit_resource_ingestion(
path=path,
ctx=ctx,
to=to,
to_is_directory=to_is_directory,
parent=parent,
reason=reason,
instruction=instruction,
wait=False,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
watch_interval=watch_interval,
manage_watch=False,
allow_local_path_resolution=allow_local_path_resolution,
enforce_public_remote_targets=enforce_public_remote_targets,
add_type=add_type,
args=args,
connector_states=connector_states,
connector_auth_config=connector_auth_config,
**kwargs,
)
async def _submit_resource_ingestion(
self,
path: str,
ctx: RequestContext,
add_type: Optional[str] = None,
to: Optional[str] = None,
to_is_directory: Optional[bool] = None,
parent: Optional[str] = None,
reason: str = "",
instruction: str = "",
wait: bool = False,
build_index: bool = True,
summarize: bool = False,
processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
watch_interval: float = 0,
is_active: Optional[bool] = None,
manage_watch: bool = True,
tags: Optional[List[str]] = None,
tag_mode: str = "replace",
parse_mode: ParseMode | str | None = None,
allow_local_path_resolution: bool = True,
enforce_public_remote_targets: bool = False,
internal_task: bool = False,
args: Optional[Dict[str, Any]] = None,
connector_states: Optional[Dict[str, Any]] = None,
connector_auth_config: Optional[Dict[str, Any]] = None,
shared_source: Optional["SharedSource"] = None,
acl: AclSpec | dict[str, Any] | None = None,
**kwargs,
) -> Dict[str, Any]:
"""Validate and route one resource ingestion request.
Watch ownership:
Native Watches require an unoccupied resolved target and keep it while
paused. Connector Watches may share targets only with other Connector
Watches; repeating a source and target creates another independent task.
Re-importing never updates or resumes an existing Watch: use
PATCH /api/v1/watches/{task_id}, or delete it before creating a replacement.
URI lookup is ambiguous when multiple accessible Watches share a target;
address those tasks by task_id. Connector Watches are visible before the
initial import and held by the scheduler until it records its result.
Args:
path: Resource path (local file or URL)
add_type: Explicitly declared Connector source type. Routes the
request to the Connector integration without probing the path;
the type must be enabled in connector.allowed_add_types. A
declared request never degrades to the standard pipeline and
requires an exact ``to`` target.
to: Exact final URI including the leaf name (e.g.,
"viking://resources/my_resource"). Written verbatim; an existing
target is synced to match the new source, so visible entries it does
not contain are deleted. Required when ``add_type`` is set.
parent: Existing directory to store the resource under; the leaf name
comes from the source. Never overwrites — a collision reserves the
next free name ("name_1", "name_2", ...) and returns a warning.
Mutually exclusive with ``to`` and not supported when ``add_type``
is set. Leaving both empty derives the directory and the name from
the source and handles collisions like ``parent``.
reason: Reason for adding the resource
instruction: Processing instruction for semantic extraction
wait: Whether to wait for semantic extraction and vectorization to complete
build_index: Whether to build vector index immediately (default: True)
summarize: Whether to generate summary (default: False)
processing_mode: Post-ingest processing mode for semantic/vector work
watch_interval: Interval in minutes (default: 0). Positive values create
a new Watch subject to target ownership rules, using explicit ``to``
or the imported ``root_uri``. Nonpositive values create no Watch:
native imports with explicit ``to`` pause a single accessible Watch
(ConflictError if ambiguous); Connector imports leave Watches untouched.
is_active: When false, the Connector, native Feishu, or native Git Watch is created paused.
Requires watch_interval > 0 and an explicit to or parent target.
enforce_public_remote_targets: When True, reject non-public remote hosts and
validate each outbound HTTP request URL during fetch.
args: Parser/accessor-specific options forwarded to the processing chain.
**kwargs: Extra options forwarded to the parser chain
Returns:
Processing result containing 'root_uri' and other metadata
Raises:
ConflictError: Incompatible target occupancy or ambiguous cancellation by URI
InvalidArgumentError: If the URI scope is not 'resources'
"""
self._ensure_initialized()
has_target = bool((to or "").strip() or (parent or "").strip())
if is_active is False and (watch_interval <= 0 or not has_target):
raise InvalidArgumentError(
"is_active=false requires watch_interval > 0 and either 'to' or 'parent'."
)
processing_mode = normalize_processing_mode(processing_mode)
self._validate_add_resource_tag_policy(tags=tags, tag_mode=tag_mode)
from openviking.connector.delegate import (
CONNECTOR_DISPLAY_RESOURCE_CONFIG_ARG,
ConnectorDelegate,
)
allowed_reserved_fields = ConnectorDelegate.supported_args(path, add_type).intersection(
_ADD_RESOURCE_ARGS_RESERVED_FIELDS
)
normalized_args = await self._normalize_add_resource_args(
args,
ctx=ctx,
watch_interval=watch_interval,
allowed_reserved_fields=allowed_reserved_fields,
)
mode = (
normalize_parse_mode(parse_mode)
if parse_mode is not None
else normalized_args.parse_mode
)
duplicated_fields = sorted(
field
for field in allowed_reserved_fields
if field in normalized_args.processor_kwargs and kwargs.get(field) is not None
)
if duplicated_fields:
raise InvalidArgumentError(
f"{', '.join(duplicated_fields)} cannot be provided both as a top-level "
"field and in args."
)
kwargs.update(normalized_args.processor_kwargs)
tos_signature = kwargs.get("tos_signature")
tos_access = kwargs.get("tos_access")
if tos_signature is not None or tos_access is not None:
if not path.startswith(("http://", "https://")):
raise InvalidArgumentError(
"tos_signature and tos_access are only supported for HTTP(S) resource URLs."
)
if tos_signature is not None and tos_access is not None:
raise InvalidArgumentError("tos_signature and tos_access cannot both be provided.")
for field, value in (("tos_signature", tos_signature), ("tos_access", tos_access)):
if value is not None:
if not isinstance(value, str) or not value.strip():
raise InvalidArgumentError(f"args.{field} must be a non-empty string.")
kwargs[field] = value.strip()
git_repo_source = is_git_repo_url(path)
if git_repo_source:
reject_git_http_userinfo(path)
if watch_interval > 0 and kwargs.get("temp_file_id"):
# Fail fast: a watch on a static upload snapshot can never observe the
# live source.
raise InvalidArgumentError(
"watch_interval > 0 is not supported for uploaded content: an "
"upload is a static snapshot, so the watch would re-process "
"stale content forever. Watch a URL / "
"sitemap / RSS source instead, or re-add the resource when the "
"source changes."
)
if not to and not parent:
from openviking.server.dependencies import get_server_config
default_parent = await effective_resource_add_target(
viking_fs=self._viking_fs,
ctx=ctx,
server_config=get_server_config(),
)
if default_parent:
parent = default_parent
kwargs["create_parent"] = True
self._ensure_single_resource_target(to=to, parent=parent)
target_to = to or ""
target_parent = parent or ""
await self._ensure_resource_target_available(
target_to, ctx, is_parent=is_content_root_uri(target_to, kind="resource")
)
await self._ensure_resource_target_available(target_parent, ctx, is_parent=True)
target_create_parent = bool(kwargs.get("create_parent", False))
if acl is not None:
acl = AclSpec.model_validate(acl)
connector = self._connector
delegate_to_connector = connector.should_delegate(
path,
ctx=ctx,
declared_add_type=add_type,
to=to,
parent=parent,
wait=wait,
instruction=instruction,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
parse_mode=mode,
watch_interval=watch_interval,
connector_args=normalized_args.processor_kwargs,
kwargs=kwargs,
)
if not delegate_to_connector and connector_auth_config:
# A managed credential reference only resolves inside Connector
# delegation; never drop it and silently degrade to an unauthorized
# standard-pipeline import.
from openviking.connector.auth import extract_credential_reference
if extract_credential_reference(connector_auth_config) is not None:
raise InvalidArgumentError(
"connector_auth_config requires a Connector import; the source or "
"parameters did not route to the Connector integration."
)
if delegate_to_connector:
resolved = connector.resolve_add_type(path, add_type)
if resolved is None: # pragma: no cover - should_delegate already resolved it
raise InvalidArgumentError(f"'{path}' does not match any Connector source type.")
acl_update = (
await self._viking_fs.prepare_acl_update(target_to, acl, ctx)
if acl is not None
else None
)
watch_manager = self._get_watch_manager()
watch_auth_state = None
create_watch = bool(watch_manager and manage_watch and watch_interval > 0)
if create_watch:
connector_watch_auth_args = dict(normalized_args.processor_kwargs)
connector_watch_auth_args.pop(CONNECTOR_DISPLAY_RESOURCE_CONFIG_ARG, None)
watch_auth_state = await connector.create_watch_auth_state(
api_key=ctx.api_key or "",
account_id=ctx.account_id,
add_type=resolved[0],
path=path,
connector_args=connector_watch_auth_args,
connector_auth_config=connector_auth_config,
)
connector_watch_processor_kwargs = self._watch_processor_kwargs(
{
key: value
for key, value in kwargs.items()
if key not in normalized_args.processor_kwargs
},
tags,
tag_mode,
)
resource_config = _normalized_resource_config(
normalized_args.processor_kwargs.get(CONNECTOR_DISPLAY_RESOURCE_CONFIG_ARG)
)
if resource_config:
connector_watch_processor_kwargs[CONNECTOR_DISPLAY_RESOURCE_CONFIG_ARG] = (
resource_config
)
on_success: Optional[Callable[[Optional[Dict[str, Any]]], Awaitable[None]]] = None
on_complete: Optional[
Callable[[str, Optional[str], Optional[str]], Awaitable[None]]
] = None
if create_watch:
# The Watch exists before the import runs so it is visible at once.
# A scheduler hold keeps the first round from overlapping with a
# scheduled run; record_watch_execution releases it at the end.
watch = await self._handle_watch_task_creation(
path=path,
to_uri=target_to,
to_is_directory=to_is_directory,
parent_uri=target_parent,
reason=reason,
instruction=instruction,
watch_interval=watch_interval,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
processor_kwargs=connector_watch_processor_kwargs,
auth_state=watch_auth_state,
connector_states=None,
source_type=resolved[0],
ctx=ctx,
is_active=is_active is not False,
hold_execution=True,
)
if watch is None:
raise InternalError("Failed to create Connector watch task.")
watch_task_id = watch.task_id
logger.info(
"[ResourceService] Connector watch ready before import: watch_task_id=%s "
"to=%s add_type=%s is_active=%s",
watch_task_id,
target_to,
resolved[0],
watch.is_active,
)
async def record_first_run(
status: str,
execution_task_id: Optional[str],
error: Optional[str],
) -> None:
try:
await self.record_watch_execution(
watch_task_id,
status=status,
execution_task_id=execution_task_id,
error=error,
)
except Exception:
logger.exception(
"[ResourceService] Failed to record Connector watch result"
)
async def store_connector_states(
new_connector_states: Optional[Dict[str, Any]] = None,
) -> None:
if new_connector_states is not None:
await watch_manager.update_connector_states(
watch_task_id,
new_connector_states,
)
on_complete = record_first_run
on_success = store_connector_states
if acl_update is not None:
watch_on_success = on_success
async def apply_connector_acl(new_states=None):
await self._viking_fs.set_acl(
acl_update.uri,
acl_update.acl.entries,
ctx=ctx,
acl_mode=acl_update.acl.acl_mode,
)
if watch_on_success is not None:
await watch_on_success(new_states)
on_success = apply_connector_acl
try:
result = await connector.submit(
path=path,
ctx=ctx,
declared_add_type=add_type,
to=target_to,
reason=reason,
connector_args=normalized_args.processor_kwargs,
connector_auth_config=connector_auth_config,
tags=tags,
tag_mode=tag_mode,
wait_for_completion=not manage_watch and watch_interval > 0,
connector_states=connector_states,
on_success=on_success,
on_complete=on_complete,
**kwargs,
)
except asyncio.CancelledError:
if on_complete is not None:
await on_complete(
"failed",
None,
"connector task submission cancelled",
)
raise
except Exception as exc:
if on_complete is not None:
await on_complete("failed", None, str(exc))
raise
# A one-off Connector import (watch_interval <= 0) leaves any watch on the
# target alone: many imports share one folder, so the native
# "watch_interval=0 cancels the watch" rule would pause it as a side
# effect. Connector watches are paused or deleted only via the watches API.
if create_watch:
result["watch_id"] = watch_task_id
return result
from openviking.parse.accessors.feishu_accessor import FeishuAccessor
if is_active is False and not (
FeishuAccessor._is_feishu_url(path) or is_git_repo_url(path)
):
raise InvalidArgumentError(
"is_active=false is only supported for Connector, native Feishu, or native Git imports."
)
if enforce_public_remote_targets and is_remote_resource_source(path):
path = require_remote_resource_source(path)
kwargs.setdefault("request_validator", ensure_public_remote_target)
async with feishu_auth_scope(
connector, path=path, ctx=ctx, args=kwargs, state=normalized_args.watch_auth_state
) as watch_auth_state:
normalized_args.watch_auth_state = watch_auth_state
source_plan = await self._prepare_standard_source_plan(
path=path,
ctx=ctx,
mode=mode,
allow_local_path_resolution=allow_local_path_resolution,
processor_kwargs=kwargs,
watch_auth_state=normalized_args.watch_auth_state,
shared_source=shared_source,
)
if source_plan is not None:
result = await self._enqueue_source_plan(
source_plan,
ctx=ctx,
to=target_to,
parent=target_parent,
create_parent=target_create_parent,
reason=reason,
instruction=instruction,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
mode=mode,
watch_interval=watch_interval,
manage_watch=manage_watch,
tags=tags,
tag_mode=tag_mode,
acl=acl,
to_is_directory=to_is_directory,
allow_local_path_resolution=allow_local_path_resolution,
enforce_public_remote_targets=enforce_public_remote_targets,
processor_kwargs=kwargs,
internal_task=internal_task,
is_active=is_active,
)
else:
result = await self._execute_resource_ingestion(
path=path,
ctx=ctx,
to=target_to,
to_is_directory=to_is_directory,
parent=target_parent,
reason=reason,
instruction=instruction,
defer_post_processing=True,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
parse_mode=mode,
watch_interval=watch_interval,
is_active=is_active,
manage_watch=manage_watch,
tags=tags,
tag_mode=tag_mode,
allow_local_path_resolution=allow_local_path_resolution,
enforce_public_remote_targets=enforce_public_remote_targets,
watch_auth_state=normalized_args.watch_auth_state,
internal_task=internal_task,
acl=acl,
**kwargs,
)
get_current_telemetry().set("resource.flags.wait", wait)
if not wait:
return result
if result.get("status") == "error":
return result
from openviking.service.task_tracker import TaskStatus, get_task_tracker
task_id = result["task_id"]
task = await get_task_tracker().wait(
task_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
if task.status == TaskStatus.COMPLETED:
completed = dict(task.result)
completed.pop("task_id", None)
return completed
if task.status == TaskStatus.CANCELLED:
return {"status": "cancelled"}
failure: Dict[str, Any] = {
"status": "error",
"errors": [task.error],
}
if isinstance(task.result, dict):
code = task.result.get("code")
if isinstance(code, str) and code:
failure["code"] = code
details = task.result.get("details")
if isinstance(details, dict):
failure["details"] = details
return failure
async def _execute_resource_ingestion(
self,
path: str,
ctx: RequestContext,
defer_post_processing: bool,
to: Optional[str] = None,
to_is_directory: Optional[bool] = None,
parent: Optional[str] = None,
reason: str = "",
instruction: str = "",
build_index: bool = True,
summarize: bool = False,
processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
parse_mode: ParseMode | str = ParseMode.DEFAULT,
watch_interval: float = 0,
is_active: Optional[bool] = None,
manage_watch: bool = True,
tags: Optional[List[str]] = None,
tag_mode: str = "replace",
allow_local_path_resolution: bool = True,
enforce_public_remote_targets: bool = False,
watch_auth_state: Optional[Dict[str, Any]] = None,
resource_lock: Optional[Dict[str, Any]] = None,
stage_callback: Optional[Callable[[str], Any]] = None,
prepared_resource: Optional["LocalResource"] = None,
internal_task: bool = False,
on_watch_ready: Optional[Callable[[str], None]] = None,
acl: AclSpec | dict[str, Any] | None = None,
**kwargs,
) -> Dict[str, Any]:
"""Execute an already-routed resource ingestion."""
self._ensure_initialized()
mode = normalize_parse_mode(parse_mode)
if mode is ParseMode.NO_SPLIT:
kwargs["parse_mode"] = mode.value
telemetry = get_current_telemetry()
telemetry_id = telemetry.telemetry_id
register_telemetry(telemetry)
job_enqueued = False
deferred_lock: Optional[Dict[str, Any]] = None
watch_manager = self._get_watch_manager()
watch_enabled = bool(watch_manager and manage_watch and watch_interval > 0)
telemetry.set("resource.flags.build_index", build_index)
telemetry.set("resource.flags.summarize", summarize)
telemetry.set("resource.flags.watch_enabled", watch_enabled)
try:
self._ensure_single_resource_target(to=to, parent=parent)
target_to = to or ""
target_parent = parent or ""
await self._ensure_resource_target_available(
target_to, ctx, is_parent=is_content_root_uri(target_to, kind="resource")
)
await self._ensure_resource_target_available(target_parent, ctx, is_parent=True)
if to_is_directory is None:
to_is_directory = bool(target_to)
watch_to_is_directory = to_is_directory
if enforce_public_remote_targets and is_remote_resource_source(path):
path = require_remote_resource_source(path)
kwargs.setdefault("request_validator", ensure_public_remote_target)
if resource_lock is not None:
kwargs["resource_lock"] = resource_lock
result = await self._resource_processor.process_resource(
path=path,
ctx=ctx,
reason=reason,
instruction=instruction,
scope="resources",
to=target_to,
parent=target_parent,
to_is_directory=to_is_directory,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
stage_callback=stage_callback,
allow_local_path_resolution=allow_local_path_resolution,
prepared_resource=prepared_resource,
target_validator=self._ensure_resource_target_available,
defer_post_processing=True,
tags=tags,
tag_mode=tag_mode,
acl=acl,
**kwargs,
)
prepared_resource = None
if result.get("status") == "error":
return result
prepared = result.pop("_post_process", None)
deferred_lock = result.pop("_resource_lock", None)
if (
not to_is_directory
and isinstance(prepared, dict)
and isinstance(prepared.get("root_is_file"), bool)
):
watch_to_is_directory = not prepared["root_is_file"]
if not isinstance(prepared, dict):
raise InternalError("Deferred resource processing payload is missing")
await self._manage_watch_if_needed(
watch_manager=watch_manager,
manage_watch=manage_watch,
watch_interval=watch_interval,
is_active=is_active,
to=target_to,
parent=target_parent,
to_is_directory=watch_to_is_directory,
root_uri=str(result.get("root_uri") or ""),
path=path,
reason=reason,
instruction=instruction,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
processor_kwargs=self._watch_processor_kwargs(kwargs, tags, tag_mode),
watch_auth_state=watch_auth_state,
ctx=ctx,
on_watch_ready=on_watch_ready,
)
if defer_post_processing:
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
root_uri = result.get("root_uri", "")
lock_handoff = await self._lock_to_handoff_payload(deferred_lock)
msg = AddResourceMsg(
task_id=str(uuid4()),
job_phase=AddResourcePhase.POST_PROCESS,
root_uri=root_uri,
prepared=prepared,
source_path=str(
(kwargs.get("source_name") or "")
if kwargs.get("temp_file_id")
else result.get("source_path") or ""
),
telemetry_id=telemetry_id or None,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
group_ids=list(ctx.group_ids),
role=str(ctx.role),
actor_peer_id=ctx.actor_peer_id,
bypass_acl=ctx.bypass_acl,
lock_handoff=lock_handoff,
reason=reason,
instruction=instruction,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
strict=bool(kwargs.get("strict", False)),
ignore_dirs=kwargs.get("ignore_dirs"),
include=kwargs.get("include"),
exclude=kwargs.get("exclude"),
directly_upload_media=bool(kwargs.get("directly_upload_media", True)),
preserve_structure=kwargs.get("preserve_structure"),
create_parent=bool(kwargs.get("create_parent", False)),
allow_local_path_resolution=allow_local_path_resolution,
enforce_public_remote_targets=enforce_public_remote_targets,
source_name=kwargs.get("source_name"),
skip_watch_management=True,
tags=tags,
tag_mode=tag_mode,
internal_task=internal_task,
)
enqueue_lock = deferred_lock
deferred_lock = None
task = await self._enqueue_add_resource_job(
msg,
queue_name=QueueManager.ADD_RESOURCE,
resource_lock=enqueue_lock,
)
result["task_id"] = task.task_id
job_enqueued = True
else:
processing_lock = deferred_lock
deferred_lock = None
post_result = await self._resource_processor.finish_prepared_resource(
prepared,
ctx=ctx,
resource_lock=processing_lock,
summarize=summarize,
build_index=build_index,
processing_mode=processing_mode,
ingest_options=IngestOptions.from_search_tags(tags, mode=tag_mode),
)
if post_result.get("warnings"):
result.setdefault("warnings", []).extend(post_result["warnings"])
return result
except Exception as exc:
telemetry.set_error(
"resource_service.add_resource",
type(exc).__name__,
str(exc),
)
raise
finally:
if prepared_resource is not None:
prepared_resource.cleanup()
if not telemetry_id or (defer_post_processing and not job_enqueued):
unregister_telemetry(telemetry_id)
if deferred_lock is not None:
await self._release_lock_ref(deferred_lock)
async def _link_resource_reason_memory(
self,
*,
result: Dict[str, Any],
ctx: RequestContext,
reason: str,
source_name: Optional[str],
) -> None:
if not self._resource_memory_link_service:
return
if not (reason or "").strip():
return
root_uri = result.get("root_uri")
if not root_uri:
return
try:
link_result = await self._resource_memory_link_service.on_resource_added(
ctx=ctx,
resource_uri=root_uri,
reason=reason,
source_name=source_name,
)
result["memory_linking"] = link_result
except Exception as exc:
logger.warning("[ResourceService] Failed to link resource reason memory: %s", exc)
result.setdefault("warnings", []).append(f"Memory linking failed: {exc}")
@staticmethod
def _raise_queue_status_errors(status: Dict[str, Any]) -> None:
failed = {
name: group
for name, group in status.items()
if isinstance(group, dict)
and (int(group.get("error_count", 0) or 0) > 0 or bool(group.get("errors")))
}
if failed:
raise InternalError(f"queue processing failed: {failed}")
# ── Connector routing ──
@property
def _connector(self) -> "ConnectorDelegate":
"""Connector delegation (lazy: viking_fs may be injected after init)."""
if self._connector_delegate is None:
from openviking.connector.delegate import ConnectorDelegate
self._connector_delegate = ConnectorDelegate(
viking_fs=self._viking_fs,
background_tasks=self._background_tasks,
link_reason_memory=self._link_resource_reason_memory,
)
return self._connector_delegate
async def _handle_watch_task_creation(
self,
path: str,
to_uri: str,
to_is_directory: bool,
parent_uri: Optional[str],
reason: str,
instruction: str,
watch_interval: float,
build_index: bool,
summarize: bool,
processing_mode: ProcessingMode,
processor_kwargs: Dict[str, Any],
auth_state: Optional[Dict[str, Any]],
connector_states: Optional[Dict[str, Any]],
source_type: Optional[str],
ctx: RequestContext,
is_active: bool = True,
hold_execution: bool = False,
) -> Optional["WatchTask"]:
"""Create the watch task for the resolved target URI.
Native Watches require an unoccupied target; Connector Watches may share
only with other Connector Watches. Paused Watches retain ownership.
Callers using ``parent`` pass the root URI resolved after ingestion.
Raises:
ConflictError: If the target URI has an incompatible Watch
"""
watch_manager = self._get_watch_manager()
if not watch_manager:
return None
task = await watch_manager.create_task(
path=path,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
original_role=str(ctx.role),
source_type=source_type,
to_uri=to_uri,
to_is_directory=to_is_directory,
parent_uri=parent_uri,
reason=reason,
instruction=instruction,
watch_interval=watch_interval,
build_index=build_index,
summarize=summarize,
processing_mode=processing_mode,
processor_kwargs=processor_kwargs,
auth_state=auth_state,
connector_states=connector_states,
is_active=is_active,
)
if hold_execution:
# Brand-new task: the hold cannot be contended, it just parks the id
# until the caller's first round records its result.
await self._hold_watch_execution(task.task_id)
logger.info(f"[ResourceService] Created watch task {task.task_id} for {to_uri}")
return task
async def _handle_watch_task_cancellation(self, to_uri: str, ctx: RequestContext) -> None:
"""Handle cancellation of watch task.
Args:
to_uri: Target URI to cancel watch for
ctx: Request context with user identity
"""
watch_manager = self._get_watch_manager()
if not watch_manager:
return
existing_task = await watch_manager.get_task_by_uri(
to_uri=to_uri,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
role=str(ctx.role),
)
if existing_task:
await watch_manager.update_task(
task_id=existing_task.task_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
role=str(ctx.role),
is_active=False,
invalidate_source_checkpoint=True,
)
logger.info(
f"[ResourceService] Deactivated watch task {existing_task.task_id} for {to_uri}"
)
async def add_skill(
self,
data: Any,
ctx: RequestContext,
wait: bool = False,
allow_local_path_resolution: bool = True,
source_path_hint: Optional[str] = None,
apply_privacy: bool = True,
privacy_change_reason: str = "auto-extracted from add_skill",
target_uri: Optional[str] = None,
source_metadata: Optional[Dict[str, Any]] = None,
task_id: Optional[str] = None,
owner_lease_ref: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Add skill to OpenViking.
Args:
data: Skill data (directory path, file path, string, or dict)
wait: Whether to wait for vectorization to complete
target_uri: Optional root URI override (e.g. ``viking://agent/skills``).
Returns:
Processing result
"""
self._ensure_initialized()
if not target_uri:
from openviking.server.dependencies import get_server_config
target_uri = await effective_skill_add_target(
viking_fs=self._viking_fs,
ctx=ctx,
server_config=get_server_config(),
)
from openviking.service.task_tracker import get_task_tracker
from openviking.service.task_tracker_concurrency import run_to_completion
from openviking.service.task_work_index import bind_task_context
task_tracker = get_task_tracker()
task = None
outcome_recorded = False
telemetry_id = get_current_telemetry().telemetry_id
telemetry_cleanup_scheduled = False
async def record_package(skill_uri: str, lease_ref: Any) -> None:
"""Preserve standalone ordinary packages under their lease for Compile rollback."""
if (
self.compile_versions is not None
and owner_lease_ref is None
and (source_metadata or {}).get("source") != "compile"
and (target := await self.compile_versions.owning_target(ctx.account_id, skill_uri))
and await self.compile_versions._write_meta(ctx.account_id, target)
):
await self.compile_versions.record_operation(
[skill_uri],
ctx,
"agent_revise",
{},
lambda: self.compile_versions.register_transfer(
skill_uri, skill_uri, ctx, lease_ref=lease_ref
),
)
try:
# Establish ownership before any queue work can start. Descendant
# semantic and embedding messages inherit this same cancellable task.
async def create_skill_task() -> None:
nonlocal task
task = await task_tracker.create(
"add_skill",
account_id=ctx.account_id,
user_id=ctx.user.user_id,
task_id=task_id,
)
# Keep the returned record before propagating cancellation, so a
# committed task can still be settled by the exception handler.
await run_to_completion(create_skill_task)
assert task is not None
await task_tracker.start(
task.task_id, account_id=ctx.account_id, user_id=ctx.user.user_id
)
running_work_id = await task_tracker.register_running_task(
task.task_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
try:
with bind_task_context(
task.task_id,
ctx.account_id,
ctx.user.user_id,
work_id=running_work_id,
):
if isinstance(data, SkillProcessingPreparation):
result = await self._skill_processor.process_prepared_skill(
data,
viking_fs=self._viking_fs,
ctx=ctx,
apply_privacy=apply_privacy,
privacy_change_reason=privacy_change_reason,
target_uri=target_uri,
source_metadata=source_metadata,
owner_lease_ref=owner_lease_ref,
on_package_written=record_package,
)
else:
result = await self._skill_processor.process_skill(
data=data,
viking_fs=self._viking_fs,
ctx=ctx,
allow_local_path_resolution=allow_local_path_resolution,
source_path_hint=source_path_hint,
apply_privacy=apply_privacy,
privacy_change_reason=privacy_change_reason,
target_uri=target_uri,
source_metadata=source_metadata,
owner_lease_ref=owner_lease_ref,
on_package_written=record_package,
)
if isinstance(result, dict) and "root_uri" not in result and result.get("uri"):
result["root_uri"] = result["uri"]
result["task_id"] = task.task_id
if telemetry_id:
self._unregister_telemetry_after_task(
task.task_id,
telemetry_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
telemetry_cleanup_scheduled = True
await task_tracker.complete(
task.task_id,
result,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
outcome_recorded = True
except BaseException as exc:
failure_message = str(exc) or "Skill processing cancelled"
await run_to_completion(
lambda: task_tracker.fail(
task.task_id,
failure_message,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
)
outcome_recorded = True
raise
finally:
await task_tracker.unregister_running_task(task.task_id)
if wait:
wait_start = time.perf_counter()
try:
completed = await task_tracker.wait(
task.task_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
timeout=None,
)
except (TimeoutError, asyncio.TimeoutError) as exc:
get_current_telemetry().set_error(
"resource_service.wait_complete",
"DEADLINE_EXCEEDED",
str(exc),
)
raise DeadlineExceededError("queue processing", None) from exc
get_current_telemetry().set(
"queue.wait.duration_ms",
round((time.perf_counter() - wait_start) * 1000, 3),
)
status = await task_tracker.build_queue_status(
task.task_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
result["queue_status"] = status
self._raise_queue_status_errors(status)
# Cancellation settles queue entries without counting errors.
if completed.status.value == "cancelled":
raise OpenVikingError(
"Skill processing was cancelled",
code="PROCESSING_ERROR",
details={"task_id": task.task_id, "status": "cancelled"},
)
return result
except BaseException as exc:
if task is not None and not outcome_recorded:
failure_message = str(exc) or "Skill processing cancelled"
await run_to_completion(
lambda: task_tracker.fail(
task.task_id,
failure_message,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
)
raise
finally:
if not telemetry_cleanup_scheduled:
unregister_telemetry(telemetry_id)
async def cancel_skill_processing(self, task_id: str, ctx: RequestContext) -> None:
"""Stop one failed update's work before its caller restores the old package."""
from openviking.service.task_tracker import get_task_tracker
tracker = get_task_tracker()
task = await tracker.cancel_skill_update_for_rollback(
task_id, account_id=ctx.account_id, user_id=ctx.user.user_id
)
if task is None:
return # Synchronous preparation failed before the task was created.
# This includes queued work that must be discarded and active handlers
# that must finish their cancellation/write cleanup, not just task status.
while await tracker.has_work(
task_id,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
):
await asyncio.sleep(0.05)
async def build_index(
self, resource_uris: List[str], ctx: RequestContext, **kwargs
) -> Dict[str, Any]:
"""Manually trigger index building.
Args:
resource_uris: List of resource URIs to index.
ctx: Request context.
Returns:
Processing result
"""
self._ensure_initialized()
return await self._resource_processor.build_index(resource_uris, ctx, **kwargs)
async def summarize(
self, resource_uris: List[str], ctx: RequestContext, **kwargs
) -> Dict[str, Any]:
"""Manually trigger summarization.
Args:
resource_uris: List of resource URIs to summarize.
ctx: Request context.
Returns:
Processing result
"""
self._ensure_initialized()
return await self._resource_processor.summarize(resource_uris, ctx, **kwargs)
async def wait_processed(self) -> Dict[str, Any]:
"""Wait for all queued processing to complete.
Returns:
Queue status
"""
qm = get_queue_manager()
status = await qm.wait_complete()
return {
name: {
"processed": s.processed,
"requeue_count": getattr(s, "requeue_count", 0),
"error_count": s.error_count,
"errors": [{"message": e.message} for e in s.errors],
}
for name, s in status.items()
}