Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf(resource): speed up wait=false add-resource (#4100)

Qin Haojie67603473a5a6
diff --git a/openviking/connector/routing.py b/openviking/connector/routing.py
index 7ff260640..74e3e7488 100644
--- a/openviking/connector/routing.py
+++ b/openviking/connector/routing.py
@@ -33,8 +33,10 @@ CONNECTOR_SUPPORTED_ARGS: Dict[str, FrozenSet[str]] = {
# request field -- never ``param_config`` -- so Connector and plugin logs
# redact them while logging param_config verbatim. The incoming OpenViking
# HTTP body may still be captured when the explicitly unsafe observability
-# body dump is enabled. One-shot use: credentials are never persisted, and
-# requests carrying them must not fall back to a durable native import job.
+# body dump is enabled. These flat fields belong only to Connector requests:
+# they are never persisted and cannot fall back to the standard Git pipeline.
+# The standard pipeline accepts its separate nested ``args.auth_config``
+# contract and stores that state only in the private task-auth field.
CONNECTOR_CREDENTIAL_ARGS: Dict[str, FrozenSet[str]] = {
"tos": frozenset(),
"git": frozenset({"token", "username"}),
diff --git a/openviking/parse/accessors/feishu_accessor.py b/openviking/parse/accessors/feishu_accessor.py
index e3dec074c..73ec41ace 100644
--- a/openviking/parse/accessors/feishu_accessor.py
+++ b/openviking/parse/accessors/feishu_accessor.py
@@ -33,6 +33,7 @@ logger = get_logger(__name__)
_FEISHU_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(feishu://image/([^)]+)\)")
_FEISHU_DOCUMENT_FORBIDDEN = 1770032
+_FEISHU_WIKI_NODE_PERMISSION_DENIED = 131006
_FEISHU_BITABLE_PERMISSION_REQUIRED = 99991672
_MAX_MEDIA_DOWNLOAD_CONTEXTS = 8
_FEISHU_DOC_PATH_TYPES = {"docx", "wiki", "sheets", "base"}
@@ -190,7 +191,7 @@ def _raise_from_lark_response(
else:
public_code = (
"PERMISSION_DENIED"
- if code == _FEISHU_DOCUMENT_FORBIDDEN
+ if code in {_FEISHU_DOCUMENT_FORBIDDEN, _FEISHU_WIKI_NODE_PERMISSION_DENIED}
else error_code_from_http_status(http_status)
)
message = f"Feishu {operation} failed: code={code}, msg={msg}"
@@ -198,6 +199,16 @@ def _raise_from_lark_response(
raise OpenVikingError(message, code=public_code, details=details)
+@dataclass(frozen=True)
+class FeishuSourcePreflight:
+ """Lightweight Feishu source identity resolved before enqueueing imports."""
+
+ doc_type: str
+ token: str
+ source_name: Optional[str]
+ source_format: str
+
+
@dataclass
class FeishuDocument:
"""Result from fetching a Feishu document."""
@@ -476,6 +487,142 @@ class FeishuAccessor(DataAccessor):
logger.error(f"[FeishuAccessor] Failed to access {source}: {e}", exc_info=True)
raise
+ async def preflight_source(
+ self,
+ source: Union[str, Path],
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> FeishuSourcePreflight:
+ """Resolve lightweight source identity and root permission before enqueueing."""
+ return await asyncio.to_thread(
+ self._preflight_source_sync,
+ str(source),
+ feishu_access_token,
+ )
+
+ def _preflight_source_sync(
+ self,
+ url: str,
+ feishu_access_token: Optional[str] = None,
+ ) -> FeishuSourcePreflight:
+ doc_type, token = self._parse_feishu_url(url)
+ query = parse_qs(urlparse(url).query)
+ table_id = (query.get("table") or [None])[0]
+ view_id = (query.get("view") or [None])[0]
+
+ if doc_type == "wiki":
+ real_type, real_token, title = self._resolve_wiki_node(
+ token,
+ feishu_access_token,
+ )
+ if real_type != "base":
+ table_id = view_id = None
+ self._probe_document_permission(
+ real_type,
+ real_token,
+ feishu_access_token=feishu_access_token,
+ table_id=table_id,
+ view_id=view_id,
+ )
+ source_name = None
+ if title:
+ scope = "/".join(value for value in (table_id, view_id) if value)
+ source_name = _title_as_filename(f"{title} ({scope})" if scope else title)
+ return FeishuSourcePreflight(
+ doc_type=real_type,
+ token=real_token,
+ source_name=source_name,
+ source_format="file",
+ )
+
+ if doc_type == "folder":
+ name = self._get_drive_folder_name(
+ token,
+ feishu_access_token=feishu_access_token,
+ )
+ self._probe_drive_folder_children(
+ token,
+ feishu_access_token=feishu_access_token,
+ )
+ return FeishuSourcePreflight(
+ doc_type=doc_type,
+ token=token,
+ source_name=_safe_path_segment(name or token, fallback=token),
+ source_format="directory",
+ )
+
+ return FeishuSourcePreflight(
+ doc_type=doc_type,
+ token=token,
+ source_name=self._preflight_document_source_name(
+ doc_type,
+ token,
+ feishu_access_token=feishu_access_token,
+ table_id=table_id,
+ view_id=view_id,
+ ),
+ source_format="file",
+ )
+
+ def _preflight_document_source_name(
+ self,
+ doc_type: str,
+ token: str,
+ *,
+ feishu_access_token: Optional[str],
+ table_id: Optional[str],
+ view_id: Optional[str],
+ ) -> Optional[str]:
+ if doc_type == "docx":
+ self._probe_docx_document(token, feishu_access_token=feishu_access_token)
+ return None
+ if doc_type == "sheets":
+ metadata = self._fetch_spreadsheet_metadata(
+ token,
+ feishu_access_token=feishu_access_token,
+ )
+ title = (metadata.get("properties") or {}).get("title") or "Spreadsheet"
+ return _title_as_filename(title)
+ if doc_type == "base":
+ if view_id and not table_id:
+ raise ValueError("Feishu Base URL with 'view' must also include 'table'")
+ if table_id:
+ self._probe_bitable_table(
+ token,
+ table_id,
+ view_id=view_id,
+ feishu_access_token=feishu_access_token,
+ )
+ return f"{table_id} ({view_id})" if view_id else table_id
+ tables = self._list_bitable_tables(
+ token,
+ feishu_access_token=feishu_access_token,
+ )
+ return f"Bitable ({len(tables)} tables)"
+ if doc_type == "file":
+ return None
+ raise ValueError(
+ f"Unsupported Feishu document type: {doc_type}. "
+ f"Supported: {list(self._DOC_TYPE_HANDLERS)}"
+ )
+
+ def _probe_document_permission(
+ self,
+ doc_type: str,
+ token: str,
+ *,
+ feishu_access_token: Optional[str],
+ table_id: Optional[str] = None,
+ view_id: Optional[str] = None,
+ ) -> None:
+ self._preflight_document_source_name(
+ doc_type,
+ token,
+ feishu_access_token=feishu_access_token,
+ table_id=table_id,
+ view_id=view_id,
+ )
+
async def _fetch_document(
self,
url: str,
@@ -759,53 +906,69 @@ class FeishuAccessor(DataAccessor):
}
)
- def _list_drive_folder_children(
+ def _fetch_drive_folder_children_page(
self,
folder_token: str,
*,
feishu_access_token: Optional[str] = None,
- ) -> List[Any]:
- """List direct children under a Feishu Drive folder token."""
+ page_token: Optional[str] = None,
+ page_size: int = 200,
+ ) -> tuple[List[Any], bool, Optional[str]]:
import lark_oapi as lark
client = self._get_client(use_user_token=bool(feishu_access_token))
token_type = (
lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT
)
- all_children: List[Any] = []
- page_token = None
+ raw_req = (
+ lark.BaseRequest.builder()
+ .http_method(lark.HttpMethod.GET)
+ .uri("/open-apis/drive/v1/files")
+ .token_types({token_type})
+ .build()
+ )
+ raw_req.add_query("folder_token", folder_token)
+ raw_req.add_query("page_size", page_size)
+ if page_token:
+ raw_req.add_query("page_token", page_token)
- while True:
- raw_req = (
- lark.BaseRequest.builder()
- .http_method(lark.HttpMethod.GET)
- .uri("/open-apis/drive/v1/files")
- .token_types({token_type})
- .build()
+ response = self._call_api(client.request, raw_req, feishu_access_token)
+ if not response.success():
+ _raise_from_lark_response(
+ response,
+ operation=f"list Drive folder {folder_token}",
+ resource=folder_token,
)
- raw_req.add_query("folder_token", folder_token)
- raw_req.add_query("page_size", 200)
- if page_token:
- raw_req.add_query("page_token", page_token)
- response = self._call_api(client.request, raw_req, feishu_access_token)
- if not response.success():
- _raise_from_lark_response(
- response,
- operation=f"list Drive folder {folder_token}",
- resource=folder_token,
- )
+ data = self._drive_response_data(response)
+ items = _getattr_safe(data, "files", None) or _getattr_safe(data, "items", None) or []
+ has_more = bool(_getattr_safe(data, "has_more", False))
+ next_page_token = _getattr_safe(data, "next_page_token", None) or _getattr_safe(
+ data,
+ "page_token",
+ None,
+ )
+ return list(items), has_more, next_page_token
- data = self._drive_response_data(response)
- items = _getattr_safe(data, "files", None) or _getattr_safe(data, "items", None) or []
+ def _list_drive_folder_children(
+ self,
+ folder_token: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> List[Any]:
+ """List direct children under a Feishu Drive folder token."""
+ all_children: List[Any] = []
+ page_token = None
+ while True:
+ items, has_more, page_token = self._fetch_drive_folder_children_page(
+ folder_token,
+ feishu_access_token=feishu_access_token,
+ page_token=page_token,
+ )
all_children.extend(items)
- has_more = bool(_getattr_safe(data, "has_more", False))
if not has_more:
break
- page_token = _getattr_safe(data, "next_page_token", None) or _getattr_safe(
- data, "page_token", None
- )
if not page_token:
raise RuntimeError(
f"Feishu returned more Drive folder items for {folder_token} "
@@ -814,6 +977,18 @@ class FeishuAccessor(DataAccessor):
return all_children
+ def _probe_drive_folder_children(
+ self,
+ folder_token: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> None:
+ self._fetch_drive_folder_children_page(
+ folder_token,
+ feishu_access_token=feishu_access_token,
+ page_size=1,
+ )
+
def _drive_folder_display_name(
self,
folder_token: str,
@@ -1187,6 +1362,35 @@ class FeishuAccessor(DataAccessor):
return markdown, doc_title
+ def _probe_docx_document(
+ self,
+ document_id: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> None:
+ """Check document block read permission without loading the whole document."""
+ from lark_oapi.api.docx.v1 import ListDocumentBlockRequest
+
+ client = self._get_client(use_user_token=bool(feishu_access_token))
+ request = (
+ ListDocumentBlockRequest.builder()
+ .document_id(document_id)
+ .page_size(1)
+ .document_revision_id(-1)
+ .build()
+ )
+ response = self._call_api(
+ client.docx.v1.document_block.list,
+ request,
+ feishu_access_token,
+ )
+ if not response.success():
+ _raise_from_lark_response(
+ response,
+ operation=f"probe document {document_id}",
+ resource=document_id,
+ )
+
def _fetch_all_blocks(
self,
document_id: str,
@@ -1724,32 +1928,11 @@ class FeishuAccessor(DataAccessor):
media_download_extras: Optional[_MediaDownloadExtras] = None,
) -> Tuple[str, str]:
"""Fetch a Feishu spreadsheet and convert it to Markdown."""
- import lark_oapi as lark
-
- client = self._get_client(use_user_token=bool(feishu_access_token))
config = self._get_config()
- token_type = (
- lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT
- )
- metadata_request = (
- lark.BaseRequest.builder()
- .http_method(lark.HttpMethod.GET)
- .uri(f"/open-apis/sheets/v2/spreadsheets/{token}/metainfo")
- .token_types({token_type})
- .build()
- )
- metadata_response = self._call_api(
- client.request,
- metadata_request,
- feishu_access_token,
+ metadata = self._fetch_spreadsheet_metadata(
+ token,
+ feishu_access_token=feishu_access_token,
)
- if not metadata_response.success():
- _raise_from_lark_response(
- metadata_response,
- operation=f"fetch spreadsheet metadata for {token}",
- resource=token,
- )
- metadata = json.loads(metadata_response.raw.content).get("data", {})
title = (metadata.get("properties") or {}).get("title") or "Spreadsheet"
sheets = metadata.get("sheets") or []
markdown_parts = [f"# {title}", f"**Sheets:** {len(sheets)}"]
@@ -1808,6 +1991,38 @@ class FeishuAccessor(DataAccessor):
return "\n\n".join(markdown_parts), title
+ def _fetch_spreadsheet_metadata(
+ self,
+ token: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> Dict[str, Any]:
+ import lark_oapi as lark
+
+ client = self._get_client(use_user_token=bool(feishu_access_token))
+ token_type = (
+ lark.AccessTokenType.USER if feishu_access_token else lark.AccessTokenType.TENANT
+ )
+ metadata_request = (
+ lark.BaseRequest.builder()
+ .http_method(lark.HttpMethod.GET)
+ .uri(f"/open-apis/sheets/v2/spreadsheets/{token}/metainfo")
+ .token_types({token_type})
+ .build()
+ )
+ metadata_response = self._call_api(
+ client.request,
+ metadata_request,
+ feishu_access_token,
+ )
+ if not metadata_response.success():
+ _raise_from_lark_response(
+ metadata_response,
+ operation=f"fetch spreadsheet metadata for {token}",
+ resource=token,
+ )
+ return json.loads(metadata_response.raw.content).get("data", {})
+
def _read_sheet_range(
self,
token: str,
@@ -1867,7 +2082,6 @@ class FeishuAccessor(DataAccessor):
from lark_oapi.api.bitable.v1 import (
ListAppTableFieldRequest,
ListAppTableRecordRequest,
- ListAppTableRequest,
)
client = self._get_client(use_user_token=bool(feishu_access_token))
@@ -1880,30 +2094,10 @@ class FeishuAccessor(DataAccessor):
markdown_parts = []
heading = "###"
else:
- table_models = []
- page_token = None
- while True:
- builder = ListAppTableRequest.builder().app_token(app_token).page_size(100)
- if page_token:
- builder = builder.page_token(page_token)
- tables_response = self._call_api(
- client.bitable.v1.app_table.list,
- builder.build(),
- feishu_access_token,
- )
- if not tables_response.success():
- _raise_from_lark_response(
- tables_response,
- operation=f"list bitable tables for {app_token}",
- resource=app_token,
- )
- table_models.extend(tables_response.data.items or [])
- if not getattr(tables_response.data, "has_more", False):
- break
- page_token = getattr(tables_response.data, "page_token", None)
- if not page_token:
- raise RuntimeError("Feishu returned more bitable tables without a page token")
-
+ table_models = self._list_bitable_tables(
+ app_token,
+ feishu_access_token=feishu_access_token,
+ )
tables = [(table.table_id, table.name or table.table_id) for table in table_models]
title = f"Bitable ({len(tables)} tables)"
markdown_parts = [f"# {title}"]
@@ -2017,6 +2211,90 @@ class FeishuAccessor(DataAccessor):
return "\n\n".join(markdown_parts), title
+ def _list_bitable_tables(
+ self,
+ app_token: str,
+ *,
+ feishu_access_token: Optional[str] = None,
+ ) -> List[Any]:
+ from lark_oapi.api.bitable.v1 import ListAppTableRequest
+
+ client = self._get_client(use_user_token=bool(feishu_access_token))
+ table_models = []
+ page_token = None
+ while True:
+ builder = ListAppTableRequest.builder().app_token(app_token).page_size(100)
+ if page_token:
+ builder = builder.page_token(page_token)
+ tables_response = self._call_api(
+ client.bitable.v1.app_table.list,
+ builder.build(),
+ feishu_access_token,
+ )
+ if not tables_response.success():
+ _raise_from_lark_response(
+ tables_response,
+ operation=f"list bitable tables for {app_token}",
+ resource=app_token,
+ )
+ table_models.extend(tables_response.data.items or [])
+ if not getattr(tables_response.data, "has_more", False):
+ break
+ page_token = getattr(tables_response.data, "page_token", None)
+ if not page_token:
+ raise RuntimeError("Feishu returned more bitable tables without a page token")
+ return table_models
+
+ def _probe_bitable_table(
+ self,
+ app_token: str,
+ table_id: str,
+ *,
+ view_id: Optional[str] = None,
+ feishu_access_token: Optional[str] = None,
+ ) -> None:
+ from lark_oapi.api.bitable.v1 import (
+ ListAppTableFieldRequest,
+ ListAppTableRecordRequest,
+ )
+
+ client = self._get_client(use_user_token=bool(feishu_access_token))
+ fields_response = self._call_api(
+ client.bitable.v1.app_table_field.list,
+ ListAppTableFieldRequest.builder()
+ .app_token(app_token)
+ .table_id(table_id)
+ .page_size(1)
+ .build(),
+ feishu_access_token,
+ )
+ if not fields_response.success():
+ _raise_from_lark_response(
+ fields_response,
+ operation=f"probe bitable fields for table {table_id}",
+ resource=app_token,
+ )
+
+ record_builder = (
+ ListAppTableRecordRequest.builder()
+ .app_token(app_token)
+ .table_id(table_id)
+ .page_size(1)
+ )
+ if view_id:
+ record_builder = record_builder.view_id(view_id)
+ records_response = self._call_api(
+ client.bitable.v1.app_table_record.list,
+ record_builder.build(),
+ feishu_access_token,
+ )
+ if not records_response.success():
+ _raise_from_lark_response(
+ records_response,
+ operation=f"probe bitable records for table {table_id}",
+ resource=app_token,
+ )
+
@classmethod
def _collect_bitable_media_extras(
cls,
diff --git a/openviking/parse/backend.py b/openviking/parse/backend.py
new file mode 100644
index 000000000..ce9f75aab
--- /dev/null
+++ b/openviking/parse/backend.py
@@ -0,0 +1,21 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Parser backend selection shared across resource-ingestion stages."""
+
+from enum import Enum
+from typing import Optional
+
+
+class ParserBackend(str, Enum):
+ INTERNAL = "internal"
+ UNDERSTANDING = "understanding"
+
+
+def normalize_parser_backend(value: object) -> Optional[ParserBackend]:
+ """Normalize an optional parser backend at a transport boundary."""
+ if value is None:
+ return None
+ try:
+ return ParserBackend(value)
+ except (TypeError, ValueError) as exc:
+ raise ValueError(f"Unknown parser backend: {value}") from exc
diff --git a/openviking/parse/parser_router.py b/openviking/parse/parser_router.py
index 5b8a1c01b..1f24ab1ef 100644
--- a/openviking/parse/parser_router.py
+++ b/openviking/parse/parser_router.py
@@ -11,6 +11,7 @@ from typing import Union
from urllib.parse import urlparse
from openviking.parse.accessors.base import LocalResource, SourceType
+from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.base import ParseResult
from openviking.parse.registry import ParserRegistry
from openviking_cli.exceptions import InvalidArgumentError
@@ -80,7 +81,10 @@ class ParserRouter:
return ext in extensions
def should_use_understanding_directly(self, source: str, **kwargs) -> bool:
- forced = kwargs.get("parser_backend") == "understanding"
+ parser_backend = normalize_parser_backend(kwargs.get("parser_backend"))
+ if parser_backend is ParserBackend.INTERNAL:
+ return False
+ forced = parser_backend is ParserBackend.UNDERSTANDING
return bool(
(forced or self.should_use_understanding_api(source))
and self._get_understanding_api().can_submit_url_directly(source, **kwargs)
@@ -103,15 +107,13 @@ class ParserRouter:
"""
source_path = self._extract_source_path(source)
- parser_backend = kwargs.pop("parser_backend", None)
- if parser_backend not in {None, "internal", "understanding"}:
- raise ValueError(f"Unknown parser backend: {parser_backend}")
+ parser_backend = normalize_parser_backend(kwargs.pop("parser_backend", None))
normalized_feishu = (
isinstance(source, LocalResource) and source.source_type == SourceType.FEISHU
)
use_understanding = not normalized_feishu and (
- parser_backend == "understanding"
+ parser_backend is ParserBackend.UNDERSTANDING
or (
parser_backend is None
and self.should_use_understanding_api(
@@ -123,8 +125,7 @@ class ParserRouter:
if use_understanding and kwargs.get("split_content") is False:
raise InvalidArgumentError(
- "parse_mode='no_split' is not supported by the configured "
- "Understanding parser."
+ "parse_mode='no_split' is not supported by the configured Understanding parser."
)
if use_understanding:
diff --git a/openviking/resource/staged_source.py b/openviking/resource/staged_source.py
new file mode 100644
index 000000000..a9eaaa130
--- /dev/null
+++ b/openviking/resource/staged_source.py
@@ -0,0 +1,199 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Task-owned source snapshots for durable resource ingestion."""
+
+import asyncio
+import os
+import shutil
+import tempfile
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any, Dict
+
+from openviking.parse.accessors.base import LocalResource, SourceType
+from openviking.server.identity import RequestContext
+from openviking.utils.path_safety import safe_join_viking_uri, sanitize_relative_viking_path
+from openviking_cli.exceptions import InvalidArgumentError
+
+_COPY_CONCURRENCY = 8
+_IDENTITY_META_FIELDS = frozenset(
+ {
+ "extension",
+ "resolved_extension",
+ "resolved_name",
+ "original_filename",
+ }
+)
+
+
+@dataclass(frozen=True)
+class StagedSource:
+ """A LocalResource copied into task-owned VikingFS temporary storage."""
+
+ temp_uri: str
+ source_uri: str
+ source_type: str
+ original_source: str
+ meta: Dict[str, Any]
+
+ def to_dict(self) -> Dict[str, Any]:
+ return asdict(self)
+
+ @classmethod
+ def from_dict(cls, data: Dict[str, Any]) -> "StagedSource":
+ if not isinstance(data, dict):
+ raise ValueError("staged_source must be an object")
+ temp_uri = data.get("temp_uri")
+ source_uri = data.get("source_uri")
+ source_type = data.get("source_type")
+ original_source = data.get("original_source")
+ meta = data.get("meta")
+ if not isinstance(temp_uri, str) or not temp_uri.startswith("viking://temp/"):
+ raise ValueError("staged_source.temp_uri must be a temp URI")
+ bundle_uri = f"{temp_uri.rstrip('/')}/source/"
+ if not isinstance(source_uri, str) or not source_uri.startswith(bundle_uri):
+ raise ValueError("staged_source.source_uri must be inside its temp bundle")
+ if source_type not in {
+ SourceType.LOCAL,
+ SourceType.GIT,
+ SourceType.HTTP,
+ SourceType.FEISHU,
+ }:
+ raise ValueError("staged_source.source_type is invalid")
+ if not isinstance(original_source, str):
+ raise ValueError("staged_source.original_source must be a string")
+ if not isinstance(meta, dict):
+ raise ValueError("staged_source.meta must be an object")
+ return cls(
+ temp_uri=temp_uri.rstrip("/"),
+ source_uri=source_uri.rstrip("/"),
+ source_type=source_type,
+ original_source=original_source,
+ meta=dict(meta),
+ )
+
+
+async def stage_source(
+ resource: LocalResource,
+ *,
+ viking_fs: Any,
+ ctx: RequestContext,
+) -> StagedSource:
+ """Copy one prepared source into task-owned VikingFS storage."""
+ path = resource.path
+ if not path.is_file() and not path.is_dir():
+ raise InvalidArgumentError("Resource source is not a regular file or directory.")
+
+ temp_uri = viking_fs.create_temp_uri(ctx=ctx).rstrip("/")
+ source_uri = safe_join_viking_uri(f"{temp_uri}/source", path.name or "resource")
+ try:
+ if path.is_dir():
+ await _copy_local_tree(path, source_uri, viking_fs, ctx)
+ else:
+ await viking_fs.write_file_bytes(
+ source_uri,
+ await asyncio.to_thread(path.read_bytes),
+ ctx=ctx,
+ )
+ except BaseException:
+ await viking_fs.delete_temp(temp_uri, ctx=ctx)
+ raise
+
+ return StagedSource(
+ temp_uri=temp_uri,
+ source_uri=source_uri,
+ source_type=resource.source_type,
+ original_source=resource.original_source,
+ meta={key: resource.meta[key] for key in _IDENTITY_META_FIELDS if key in resource.meta},
+ )
+
+
+async def materialize_source(
+ staged: StagedSource,
+ *,
+ viking_fs: Any,
+ ctx: RequestContext,
+) -> LocalResource:
+ """Restore a staged source into worker-local temporary storage."""
+ local_root = Path(tempfile.mkdtemp(prefix="ov_staged_source_"))
+ bundle_uri = f"{staged.temp_uri}/source"
+ source_relpath = staged.source_uri.removeprefix(f"{bundle_uri}/")
+ try:
+ entries = await viking_fs.tree(
+ bundle_uri,
+ output="original",
+ show_all_hidden=True,
+ node_limit=None,
+ level_limit=None,
+ ctx=ctx,
+ )
+ for entry in entries:
+ target = _local_target(local_root, entry.get("rel_path"))
+ if entry.get("isDir", False):
+ target.mkdir(parents=True, exist_ok=True)
+ continue
+ target.parent.mkdir(parents=True, exist_ok=True)
+ content = await viking_fs.read_file_bytes(str(entry["uri"]), ctx=ctx)
+ await asyncio.to_thread(target.write_bytes, content)
+
+ local_path = _local_target(local_root, source_relpath)
+ if not local_path.exists():
+ raise ValueError("Staged source is missing")
+ except BaseException:
+ shutil.rmtree(local_root, ignore_errors=True)
+ raise
+
+ meta = dict(staged.meta)
+ meta["_cleanup_path"] = str(local_root)
+ return LocalResource(
+ path=local_path,
+ source_type=staged.source_type,
+ original_source=staged.original_source,
+ meta=meta,
+ is_temporary=True,
+ )
+
+
+async def _copy_local_tree(
+ local_dir: Path,
+ target_uri: str,
+ viking_fs: Any,
+ ctx: RequestContext,
+) -> None:
+ directories = {target_uri}
+ files: list[tuple[Path, str]] = []
+ for root, dir_names, file_names in os.walk(local_dir, followlinks=False):
+ root_path = Path(root)
+ dir_names[:] = [name for name in dir_names if not (root_path / name).is_symlink()]
+ relative_root = root_path.relative_to(local_dir)
+ if relative_root.parts:
+ directories.add(safe_join_viking_uri(target_uri, relative_root.as_posix()))
+ for name in file_names:
+ local_path = root_path / name
+ if local_path.is_symlink() or not local_path.is_file():
+ continue
+ target_file_uri = safe_join_viking_uri(
+ target_uri,
+ local_path.relative_to(local_dir).as_posix(),
+ )
+ directories.add(target_file_uri.rsplit("/", 1)[0])
+ files.append((local_path, target_file_uri))
+
+ for directory_uri in sorted(directories):
+ await viking_fs.mkdir(directory_uri, exist_ok=True, ctx=ctx)
+
+ semaphore = asyncio.Semaphore(_COPY_CONCURRENCY)
+
+ async def copy_file(local_path: Path, target_file_uri: str) -> None:
+ async with semaphore:
+ content = await asyncio.to_thread(local_path.read_bytes)
+ await viking_fs.write_file_bytes(target_file_uri, content, ctx=ctx)
+
+ await asyncio.gather(*(copy_file(path, uri) for path, uri in files))
+
+
+def _local_target(root: Path, rel_path: Any) -> Path:
+ if not isinstance(rel_path, str):
+ raise ValueError("Staged source entry is missing rel_path")
+ normalized = sanitize_relative_viking_path(rel_path)
+ return root.joinpath(*Path(normalized).parts)
diff --git a/openviking/service/resource_service.py b/openviking/service/resource_service.py
index a0a96d2ec..d9e67e18f 100644
--- a/openviking/service/resource_service.py
+++ b/openviking/service/resource_service.py
@@ -12,22 +12,31 @@ import inspect
import json
import time
from collections.abc import Callable
-from dataclasses import dataclass
+from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from urllib.parse import urlparse
from uuid import uuid4
from openviking.core.content_targets import ContentTargetSpec
+from openviking.core.namespace import is_content_root_uri
from openviking.core.uri_validation import validate_optional_content_target_uri
+from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.mode import ParseMode, normalize_parse_mode
from openviking.parse.parsers.constants import MPEG_TS_EXTENSION_ALIAS
from openviking.resource.feishu_watch_auth import (
FEISHU_ACCESS_TOKEN_ARG,
+ FEISHU_AUTH_PROVIDER,
FEISHU_REFRESH_TOKEN_ARG,
create_feishu_auth_state,
+ is_feishu_auth_state,
load_feishu_app_credentials,
)
-from openviking.resource.git_watch_auth import create_git_http_auth_state
+from openviking.resource.git_watch_auth import (
+ create_git_http_auth_state,
+ git_http_auth_config_from_state,
+ is_git_http_auth_state,
+)
from openviking.resource.processing_mode import (
DEFAULT_PROCESSING_MODE,
ProcessingMode,
@@ -43,6 +52,7 @@ from openviking.server.user_config import (
effective_skill_add_target,
)
from openviking.storage.queuefs import QueueManager, get_queue_manager
+from openviking.storage.queuefs.add_resource_msg import AddResourcePhase
from openviking.storage.viking_fs import VikingFS
from openviking.storage.vikingdb_manager import VikingDBManager
from openviking.telemetry import get_current_telemetry, register_telemetry, unregister_telemetry
@@ -51,7 +61,12 @@ from openviking.telemetry.resource_summary import (
build_queue_status_payload,
)
from openviking.utils import is_git_repo_url, parse_code_hosting_url
-from openviking.utils.git_auth import parse_git_http_auth_config
+from openviking.utils.git_auth import (
+ GitHttpAuthConfig,
+ build_git_http_auth_env,
+ parse_git_http_auth_config,
+ reject_git_http_userinfo,
+)
from openviking.utils.ingest_options import IngestOptions
from openviking.utils.media_processor import _smart_stem
from openviking.utils.network_guard import ensure_public_remote_target
@@ -69,6 +84,7 @@ from openviking_cli.utils import get_logger
if TYPE_CHECKING:
from openviking.connector.delegate import ConnectorDelegate
from openviking.parse.accessors.base import LocalResource
+ from openviking.resource.staged_source import StagedSource
from openviking.resource.watch_manager import WatchManager
from openviking.resource.watch_scheduler import WatchScheduler
from openviking.service.resource_memory_link_service import ResourceMemoryLinkService
@@ -111,6 +127,7 @@ _ADD_RESOURCE_ARGS_RESERVED_FIELDS = frozenset(
"parser_backend",
"resolved_extension",
"defer_post_processing",
+ "prepared_resource",
"tags",
"tag_mode",
}
@@ -130,6 +147,7 @@ _INTERNAL_INGESTION_FIELDS = frozenset(
"understanding_response_id",
"parser_backend",
"resolved_extension",
+ "prepared_resource",
}
)
@@ -148,6 +166,17 @@ class _NormalizedAddResourceArgs:
parse_mode: ParseMode = ParseMode.DEFAULT
+@dataclass
+class _SourcePlan:
+ path: str
+ source_identity: _ResourceSourceInfo
+ processor_args: Dict[str, Any]
+ task_auth: Dict[str, Any] = field(repr=False)
+ staged_source: Optional["StagedSource"] = None
+ understanding_response_id: Optional[str] = None
+ defer_unnamed_target: bool = False
+
+
class ResourceService:
"""Resource management service."""
@@ -198,6 +227,10 @@ class ResourceService:
"auth_config",
FEISHU_ACCESS_TOKEN_ARG,
FEISHU_REFRESH_TOKEN_ARG,
+ "parser_backend",
+ "resolved_extension",
+ "understanding_response_id",
+ "temp_file_id",
}:
continue
try:
@@ -219,9 +252,6 @@ class ResourceService:
watch_kwargs["tag_mode"] = tag_mode
return watch_kwargs
- def _processor_args_for_watch_run(self, processor_kwargs: Dict[str, Any]) -> Dict[str, Any]:
- return self._sanitize_watch_processor_kwargs(processor_kwargs)
-
def _validate_add_resource_tag_policy(
self,
*,
@@ -449,17 +479,17 @@ class ResourceService:
*,
queue_name: str,
resource_lock: Optional[Dict[str, Any]] = None,
+ task_auth: Optional[Dict[str, Any]] = None,
+ on_enqueued: Optional[Callable[[], None]] = None,
) -> Any:
"""Persist a job and fully own the passed lock until handoff or release completes."""
from openviking.service.task_tracker import get_task_tracker
from openviking.storage.queuefs import get_queue_manager
tracker = get_task_tracker()
+ task = None
+ enqueued = False
try:
- await get_queue_manager().enqueue(queue_name, msg.to_dict())
- if resource_lock is not None:
- await self._handoff_lock_ref(resource_lock)
- resource_lock = None
task = await tracker.create(
"add_resource",
resource_id=None if msg.defer_target_resolution else msg.root_uri,
@@ -467,7 +497,15 @@ class ResourceService:
user_id=msg.user_id,
task_id=msg.task_id,
meta={"source_path": msg.source_path},
+ auth=task_auth,
)
+ await get_queue_manager().enqueue(queue_name, msg.to_dict())
+ enqueued = True
+ if on_enqueued is not None:
+ on_enqueued()
+ if resource_lock is not None:
+ await self._handoff_lock_ref(resource_lock)
+ resource_lock = None
await tracker.update_stage(
task.task_id,
"queued",
@@ -477,6 +515,13 @@ class ResourceService:
except BaseException:
if resource_lock is not None:
await self._release_lock_ref(resource_lock)
+ if task is not None and not enqueued:
+ await tracker.fail(
+ task.task_id,
+ "Failed to enqueue resource processing",
+ account_id=msg.account_id,
+ user_id=msg.user_id,
+ )
raise
return task
@@ -488,21 +533,57 @@ class ResourceService:
ctx: RequestContext,
resource_lock: Optional[Dict[str, Any]],
stage_callback: Callable[[str], Any],
+ task_auth: Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
"""Execute one durable add-resource job inside its QueueFS consumer."""
- if msg.prepared is None:
+ if msg.job_phase is AddResourcePhase.SOURCE:
+ from openviking.resource.staged_source import StagedSource, materialize_source
+
target_uri = msg.root_uri
parent_uri = None
- internal_kwargs: Dict[str, Any] = {}
queued_args = dict(msg.args)
- for key in ("parser_backend", "resolved_extension"):
- if key in queued_args:
- internal_kwargs[key] = queued_args.pop(key)
+ legacy_backend = normalize_parser_backend(queued_args.pop("parser_backend", None))
+ parser_backend = legacy_backend or (
+ ParserBackend.UNDERSTANDING
+ if msg.understanding_response_id is not None
+ else ParserBackend.INTERNAL
+ )
+ internal_kwargs: Dict[str, Any] = {"parser_backend": parser_backend}
+ if "resolved_extension" in queued_args:
+ internal_kwargs["resolved_extension"] = queued_args.pop("resolved_extension")
normalized_args = self._normalize_add_resource_args(
queued_args,
watch_interval=msg.watch_interval,
)
internal_kwargs.update(normalized_args.processor_kwargs)
+ if msg.strict:
+ internal_kwargs["strict"] = True
+ if msg.ignore_dirs is not None:
+ internal_kwargs["ignore_dirs"] = msg.ignore_dirs
+ if msg.include is not None:
+ internal_kwargs["include"] = msg.include
+ if msg.exclude is not None:
+ internal_kwargs["exclude"] = msg.exclude
+ if not msg.directly_upload_media:
+ internal_kwargs["directly_upload_media"] = False
+ if msg.preserve_structure is not None:
+ internal_kwargs["preserve_structure"] = msg.preserve_structure
+ if msg.create_parent:
+ internal_kwargs["create_parent"] = True
+ if msg.source_name is not None:
+ internal_kwargs["source_name"] = msg.source_name
+ auth_kwargs, watch_auth_state = self._restore_source_task_auth(
+ msg,
+ task_auth or {},
+ )
+ internal_kwargs.update(auth_kwargs)
+ prepared_resource = None
+ if msg.staged_source is not None:
+ prepared_resource = await materialize_source(
+ StagedSource.from_dict(msg.staged_source),
+ viking_fs=self._viking_fs,
+ ctx=ctx,
+ )
if msg.defer_target_resolution:
from openviking_cli.utils.uri import VikingURI
@@ -517,6 +598,7 @@ class ResourceService:
ctx=ctx,
to=target_uri,
parent=parent_uri,
+ to_is_directory=msg.to_is_directory,
reason=msg.reason,
instruction=msg.instruction,
defer_post_processing=False,
@@ -533,17 +615,12 @@ class ResourceService:
enforce_public_remote_targets=msg.enforce_public_remote_targets,
resource_lock=resource_lock,
stage_callback=stage_callback,
- watch_auth_state=normalized_args.watch_auth_state,
- strict=msg.strict,
- source_name=msg.source_name,
- ignore_dirs=msg.ignore_dirs,
- include=msg.include,
- exclude=msg.exclude,
- directly_upload_media=msg.directly_upload_media,
- preserve_structure=msg.preserve_structure,
- create_parent=msg.create_parent,
+ watch_auth_state=watch_auth_state,
+ prepared_resource=prepared_resource,
**internal_kwargs,
)
+ if msg.staged_source is not None:
+ result["source_path"] = msg.source_path
stage_result = stage_callback("processing_queue")
if inspect.isawaitable(stage_result):
await stage_result
@@ -565,6 +642,39 @@ class ResourceService:
),
)
+ def _restore_source_task_auth(
+ self,
+ msg: Any,
+ task_auth: Dict[str, Any],
+ ) -> tuple[Dict[str, Any], Optional[Dict[str, Any]]]:
+ """Restore provider-specific request inputs from task-owned auth state."""
+ if not task_auth:
+ return {}, None
+ if is_git_http_auth_state(task_auth):
+ auth_config = git_http_auth_config_from_state(task_auth, msg.path)
+ watch_auth_state = dict(task_auth) if msg.watch_interval > 0 else None
+ return {"auth_config": auth_config}, watch_auth_state
+ if is_feishu_auth_state(task_auth):
+ token = task_auth.get("access_token")
+ if not isinstance(token, str) or not token.strip():
+ raise InvalidArgumentError("Stored Feishu task credentials are invalid.")
+ if msg.watch_interval > 0:
+ refresh_token = task_auth.get("refresh_token")
+ if not isinstance(refresh_token, str) or not refresh_token.strip():
+ raise InvalidArgumentError(
+ "Stored Feishu watch credentials are missing a refresh token."
+ )
+ watch_auth_state = dict(task_auth)
+ else:
+ watch_auth_state = None
+ auth_kwargs = (
+ {FEISHU_ACCESS_TOKEN_ARG: token.strip()}
+ if msg.understanding_response_id is None
+ else {}
+ )
+ return auth_kwargs, watch_auth_state
+ raise InvalidArgumentError("Unsupported task authentication provider.")
+
async def reacquire_add_resource_job_lock(
self,
root_uri: str,
@@ -580,100 +690,243 @@ class ResourceService:
timeout_secs=0.0,
)
- async def enqueue_git_add_resource(
+ async def _prepare_standard_source_plan(
self,
+ *,
path: str,
ctx: RequestContext,
- to: Optional[str] = None,
- parent: Optional[str] = None,
- reason: str = "",
- instruction: str = "",
- timeout: Optional[float] = None,
- build_index: bool = True,
- summarize: bool = False,
- processing_mode: ProcessingMode = DEFAULT_PROCESSING_MODE,
- parse_mode: ParseMode | str | None = None,
- watch_interval: float = 0,
- manage_watch: bool = True,
- tags: Optional[List[str]] = None,
- tag_mode: str = "replace",
- allow_local_path_resolution: bool = True,
- enforce_public_remote_targets: bool = False,
- args: Optional[Dict[str, Any]] = None,
- **kwargs,
- ) -> Dict[str, Any]:
- """Start background ingestion for Git repositories while reserving the target URI."""
- self._ensure_initialized()
- processing_mode = normalize_processing_mode(processing_mode)
- self._validate_add_resource_tag_policy(tags=tags, tag_mode=tag_mode)
- normalized_args = self._normalize_add_resource_args(args, watch_interval=watch_interval)
- mode = (
- normalize_parse_mode(parse_mode)
- if parse_mode is not None
- else normalized_args.parse_mode
- )
- kwargs.update(normalized_args.processor_kwargs)
- if "auth_config" in kwargs:
- raise InvalidArgumentError(
- "args.auth_config cannot be used with enqueue_git_add_resource because "
- "native Git credentials must be consumed before durable queue submission. "
- "Call add_resource instead."
- )
- from openviking.connector.routing import credential_arg_names
+ mode: ParseMode,
+ allow_local_path_resolution: bool,
+ processor_kwargs: Dict[str, Any],
+ watch_auth_state: Optional[Dict[str, Any]],
+ ) -> Optional[_SourcePlan]:
+ """Freeze one durable standard-pipeline source before it crosses QueueFS."""
+ from openviking.parse.accessors.feishu_accessor import FeishuAccessor
+ from openviking.resource.staged_source import stage_source
+
+ source_name = processor_kwargs.get("source_name")
+ git_source = is_git_repo_url(path)
+ feishu_source = FeishuAccessor._is_feishu_url(path)
+ remote_source = is_remote_resource_source(path)
+ local_source = False
+ if allow_local_path_resolution and len(path) <= 1024 and "\n" not in path:
+ with contextlib.suppress(OSError, ValueError):
+ local_source = Path(path).exists()
+ if not (git_source or feishu_source or remote_source or local_source):
+ return None
- credential_args = credential_arg_names("git", kwargs)
- if credential_args:
- raise InvalidArgumentError(
- "Git credential args "
- f"{credential_args} cannot be used with the native asynchronous import "
- "pipeline because it persists job parameters. Use a Connector-compatible "
- "request instead."
+ queued_args = {
+ key: value
+ for key, value in processor_kwargs.items()
+ if key not in _ADD_RESOURCE_ARGS_RESERVED_FIELDS
+ }
+ queued_args = self._sanitize_watch_processor_kwargs(queued_args)
+ task_auth: Dict[str, Any] = {}
+ staged_source = None
+ understanding_response_id = None
+ defer_unnamed_target = False
+
+ if git_source:
+ reject_git_http_userinfo(path)
+ from openviking.connector.routing import credential_arg_names
+
+ credential_args = credential_arg_names("git", processor_kwargs)
+ if credential_args:
+ raise InvalidArgumentError("Native Git credentials must use args.auth_config.")
+ git_auth = parse_git_http_auth_config(processor_kwargs.get("auth_config"), path)
+ if git_auth is not None:
+ task_auth = create_git_http_auth_state(git_auth, path)
+ source_info = await self._preflight_git_source(path, auth_config=git_auth)
+ source_name = source_name or source_info.source_name
+ source_info.source_name = source_name
+ elif feishu_source:
+ token = processor_kwargs.get(FEISHU_ACCESS_TOKEN_ARG)
+ if isinstance(token, str) and token.strip():
+ task_auth = dict(
+ watch_auth_state
+ or {
+ "provider": FEISHU_AUTH_PROVIDER,
+ "access_token": token.strip(),
+ }
+ )
+ preflight = await FeishuAccessor().preflight_source(
+ path,
+ feishu_access_token=token.strip() if isinstance(token, str) else None,
+ )
+ source_name = source_name or preflight.source_name
+ source_info = _ResourceSourceInfo(
+ source_name=source_name,
+ source_path=path,
+ source_format=preflight.source_format,
+ )
+ defer_unnamed_target = True
+ direct_understanding = bool(
+ mode is ParseMode.DEFAULT
+ and self._resource_processor.should_use_understanding_directly(
+ path,
+ **processor_kwargs,
+ )
+ )
+ if direct_understanding:
+ understanding_response_id = await self._resource_processor.submit_understanding(
+ path,
+ **processor_kwargs,
+ )
+ if watch_auth_state is None:
+ task_auth = {}
+ else:
+ prepared = await self._resource_processor.prepare_durable_source(
+ path,
+ ctx,
+ snapshot_required=local_source,
+ parse_mode=mode,
+ allow_local_path_resolution=allow_local_path_resolution,
+ **processor_kwargs,
)
+ if prepared is None:
+ parsed_path = Path(urlparse(path).path)
+ source_name = source_name or parsed_path.name or None
+ source_info = _ResourceSourceInfo(
+ source_name=source_name,
+ source_path=path,
+ source_format=parsed_path.suffix.lower().lstrip(".") or "file",
+ )
+ else:
+ try:
+ resolved_extension, source_info = self._prepared_source_info(
+ prepared,
+ path,
+ source_name,
+ use_path_name=local_source,
+ )
+ if resolved_extension:
+ queued_args["resolved_extension"] = resolved_extension
+ if (
+ mode is ParseMode.DEFAULT
+ and self._resource_processor.should_use_understanding_api(prepared)
+ ):
+ understanding_response_id = (
+ await self._resource_processor.submit_understanding(
+ prepared,
+ **processor_kwargs,
+ )
+ )
+ else:
+ staged_source = await stage_source(
+ prepared,
+ viking_fs=self._viking_fs,
+ ctx=ctx,
+ )
+ finally:
+ prepared.cleanup()
- target = ContentTargetSpec.from_fields(
- ctx=ctx,
- kind="resource",
- to=to,
- parent=parent,
- create_parent=bool(kwargs.get("create_parent", False)),
+ return _SourcePlan(
+ path=path,
+ source_identity=source_info,
+ processor_args=queued_args,
+ task_auth=task_auth,
+ staged_source=staged_source,
+ understanding_response_id=understanding_response_id,
+ defer_unnamed_target=defer_unnamed_target,
)
+ @staticmethod
+ def _prepared_source_info(
+ resource: "LocalResource",
+ path: str,
+ source_name: Optional[str],
+ *,
+ use_path_name: bool = False,
+ ) -> tuple[str, _ResourceSourceInfo]:
+ resolved_extension = str(
+ resource.meta.get("resolved_extension")
+ or resource.meta.get("extension")
+ or resource.path.suffix
+ or ""
+ )
+ resolved_name = (
+ source_name
+ or resource.meta.get("original_filename")
+ or resource.meta.get("resolved_name")
+ )
+ if not resolved_name and use_path_name:
+ resolved_name = resource.path.name
+ source_format = "directory" if resource.path.is_dir() else resolved_extension.lstrip(".")
+ if not source_format:
+ source_format = "file"
+ if resolved_extension.lower().lstrip(".") == MPEG_TS_EXTENSION_ALIAS:
+ source_format = "video"
+ return (
+ resolved_extension,
+ _ResourceSourceInfo(
+ source_name=resolved_name,
+ source_path=resource.original_source or path,
+ source_format=source_format,
+ ),
+ )
+
+ async def _enqueue_source_plan(
+ self,
+ plan: _SourcePlan,
+ *,
+ ctx: RequestContext,
+ target: ContentTargetSpec,
+ reason: str,
+ instruction: str,
+ timeout: Optional[float],
+ build_index: bool,
+ summarize: bool,
+ processing_mode: ProcessingMode,
+ mode: ParseMode,
+ watch_interval: float,
+ manage_watch: bool,
+ tags: Optional[List[str]],
+ tag_mode: str,
+ to_is_directory: Optional[bool],
+ allow_local_path_resolution: bool,
+ enforce_public_remote_targets: bool,
+ processor_kwargs: Dict[str, Any],
+ ) -> Dict[str, Any]:
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
- resource_lock: Optional[Dict[str, Any]] = None
+ planned_to_is_directory = (
+ to_is_directory if to_is_directory is not None else bool(target.to)
+ )
+ target_to_is_exact = bool(
+ target.to and not is_content_root_uri(target.to, ctx, kind="resource")
+ )
+ defer_candidate_resolution = bool(
+ (
+ plan.defer_unnamed_target
+ and plan.source_identity.source_name is None
+ and not target.to
+ )
+ or (mode is ParseMode.NO_SPLIT and not target_to_is_exact)
+ )
+ root_uri = ""
+ resource_lock = None
+ defer_target_resolution = False
+ staged_enqueued = False
try:
- if enforce_public_remote_targets and is_remote_resource_source(path):
- path = require_remote_resource_source(path)
- kwargs.setdefault("request_validator", ensure_public_remote_target)
-
- source_info = await self._preflight_git_source(path)
- source_name = kwargs.get("source_name") or source_info.source_name
- if source_name:
- kwargs["source_name"] = source_name
- root_uri, resource_lock = await self._plan_resource_target(
- path=path,
+ root_uri, resource_lock, defer_target_resolution = await self._plan_source_job_target(
+ path=plan.path,
ctx=ctx,
target=target,
- source_name=source_name,
- source_info=source_info,
- )
-
- task_id = str(uuid4())
- lock_handoff = (
- await self._viking_fs._async_agfs.pathlock_to_handoff(resource_lock)
- if resource_lock is not None
- else None
+ source_info=plan.source_identity,
+ defer_candidate_resolution=defer_candidate_resolution,
)
- processor_args = {
- key: value
- for key, value in kwargs.items()
- if key not in _ADD_RESOURCE_ARGS_RESERVED_FIELDS
- }
+ lock_handoff = await self._lock_to_handoff_payload(resource_lock)
msg = AddResourceMsg(
- task_id=task_id,
- path=path,
- source_path=(source_name or "") if kwargs.get("temp_file_id") else path,
+ task_id=str(uuid4()),
+ job_phase=AddResourcePhase.SOURCE,
+ path=plan.path,
+ source_path=(plan.source_identity.source_name or "")
+ if processor_kwargs.get("temp_file_id")
+ else plan.source_identity.source_path or plan.path,
root_uri=root_uri,
+ staged_source=(
+ plan.staged_source.to_dict() if plan.staged_source is not None else None
+ ),
telemetry_id=get_current_telemetry().telemetry_id or None,
account_id=ctx.account_id,
user_id=ctx.user.user_id,
@@ -691,49 +944,70 @@ class ResourceService:
skip_watch_management=not manage_watch,
tags=tags,
tag_mode=tag_mode,
- allow_local_path_resolution=allow_local_path_resolution,
- enforce_public_remote_targets=enforce_public_remote_targets,
- strict=bool(kwargs.get("strict", False)),
- ignore_dirs=kwargs.get("ignore_dirs"),
- include=kwargs.get("include"),
- exclude=kwargs.get("exclude"),
- directly_upload_media=bool(kwargs.get("directly_upload_media", True)),
- preserve_structure=kwargs.get("preserve_structure"),
- create_parent=bool(kwargs.get("create_parent", False)),
- source_name=source_name,
- args=self._processor_args_for_watch_run(processor_args),
+ allow_local_path_resolution=(
+ True if plan.staged_source is not None else allow_local_path_resolution
+ ),
+ enforce_public_remote_targets=(
+ enforce_public_remote_targets and plan.staged_source is None
+ ),
+ strict=bool(processor_kwargs.get("strict", False)),
+ ignore_dirs=processor_kwargs.get("ignore_dirs"),
+ include=processor_kwargs.get("include"),
+ exclude=processor_kwargs.get("exclude"),
+ directly_upload_media=bool(processor_kwargs.get("directly_upload_media", True)),
+ preserve_structure=processor_kwargs.get("preserve_structure"),
+ create_parent=bool(processor_kwargs.get("create_parent", False)),
+ source_name=plan.source_identity.source_name,
+ to_is_directory=planned_to_is_directory,
+ args=plan.processor_args,
+ defer_target_resolution=defer_target_resolution,
+ understanding_response_id=plan.understanding_response_id,
+ )
+
+ def transfer_staged_source() -> None:
+ nonlocal staged_enqueued
+ staged_enqueued = True
+
+ queue_name = (
+ QueueManager.EXTERNAL_PARSE
+ if plan.understanding_response_id is not None
+ else QueueManager.ADD_RESOURCE
)
enqueue_lock = resource_lock
resource_lock = None
task = await self._enqueue_add_resource_job(
msg,
- queue_name=QueueManager.ADD_RESOURCE,
+ queue_name=queue_name,
resource_lock=enqueue_lock,
+ task_auth=plan.task_auth,
+ on_enqueued=(transfer_staged_source if plan.staged_source is not None else None),
)
- return {
- "status": "success",
- "root_uri": root_uri,
- "task_id": task.task_id,
- }
- except Exception:
+ except BaseException:
if resource_lock is not None:
- await self._viking_fs._async_agfs.pathlock_release(resource_lock)
+ await self._release_lock_ref(resource_lock)
+ if plan.staged_source is not None and not staged_enqueued:
+ await self._viking_fs.delete_temp(plan.staged_source.temp_uri, ctx=ctx)
raise
- async def _plan_resource_target(
+ response = {"status": "success", "task_id": task.task_id}
+ if not defer_target_resolution:
+ response["root_uri"] = root_uri
+ return response
+
+ async def _plan_source_job_target(
self,
*,
path: str,
ctx: RequestContext,
target: ContentTargetSpec,
- source_name: Optional[str],
source_info: _ResourceSourceInfo,
- ) -> tuple[str, Optional[Dict[str, Any]]]:
+ defer_candidate_resolution: bool,
+ ) -> tuple[str, Optional[Dict[str, Any]], bool]:
if not self._resource_processor or not self._viking_fs:
raise NotInitializedError("ResourceProcessor")
- doc_name = self._target_doc_name(path, source_name, source_info)
- source_path = source_info.source_path or source_name or path
+ doc_name = self._target_doc_name(path, source_info)
+ source_path = source_info.source_path or source_info.source_name or path
root_uri, candidate_uri = await self._resource_processor.tree_builder.resolve_target_uri(
ctx=ctx,
doc_name=doc_name,
@@ -744,27 +1018,27 @@ class ResourceService:
source_format=source_info.source_format,
create_parent=target.create_parent,
)
+ if candidate_uri and defer_candidate_resolution:
+ return root_uri, None, True
if candidate_uri:
- return await self._resource_processor.reserve_unique_candidate(
+ root_uri, resource_lock = await self._resource_processor.reserve_unique_candidate(
candidate_uri=candidate_uri,
ctx=ctx,
)
+ return root_uri, resource_lock, False
dst_path = self._viking_fs._uri_to_path(root_uri, ctx=ctx)
resource_lock = await self._viking_fs._async_agfs.pathlock_acquire_tree(
dst_path,
timeout_secs=0.0,
)
- return root_uri, resource_lock
+ return root_uri, resource_lock, False
@staticmethod
def _target_doc_name(
path: str,
- source_name: Optional[str],
source_info: _ResourceSourceInfo,
) -> str:
- if source_name:
- return _smart_stem(source_name)
if source_info.source_name:
return _smart_stem(source_info.source_name)
if source_info.source_format == "repository":
@@ -773,32 +1047,38 @@ class ResourceService:
return parsed.rsplit("/", 1)[-1]
return _smart_stem(Path(path).name or "resource")
- async def _preflight_git_source(self, source: str) -> _ResourceSourceInfo:
+ async def _preflight_git_source(
+ self,
+ source: str,
+ *,
+ auth_config: Optional[GitHttpAuthConfig] = None,
+ ) -> _ResourceSourceInfo:
+ proc = None
try:
+ env = build_git_http_auth_env(auth_config, source) if auth_config is not None else None
proc = await asyncio.create_subprocess_exec(
"git",
"ls-remote",
"--heads",
source,
+ env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=10.0)
except TimeoutError as exc:
- with contextlib.suppress(Exception):
- proc.kill() # type: ignore[possibly-undefined]
- await proc.communicate() # type: ignore[possibly-undefined]
+ if proc is not None:
+ with contextlib.suppress(Exception):
+ proc.kill()
+ await proc.communicate()
raise InvalidArgumentError(
- f"Cannot access Git repository: {source}. The check timed out after 10s."
+ "Cannot access Git repository; the preflight timed out after 10s."
) from exc
except Exception as exc:
- raise InvalidArgumentError(f"Cannot access Git repository: {source}. {exc}") from exc
+ raise InvalidArgumentError("Cannot access Git repository during preflight.") from exc
if proc.returncode != 0:
- detail = (stderr or stdout).decode("utf-8", errors="replace").strip()
- raise InvalidArgumentError(
- f"Cannot access Git repository: {source}. {detail or 'git ls-remote failed'}"
- )
+ raise InvalidArgumentError("Cannot access Git repository; git ls-remote failed.")
repo_name = parse_code_hosting_url(source)
return _ResourceSourceInfo(
source_name=repo_name.rsplit("/", 1)[-1] if repo_name else None,
@@ -979,6 +1259,8 @@ class ResourceService:
)
kwargs.update(normalized_args.processor_kwargs)
git_repo_source = is_git_repo_url(path)
+ if git_repo_source:
+ reject_git_http_userinfo(path)
if watch_interval > 0 and kwargs.get("temp_file_id"):
# Fail fast: a watch on a static upload snapshot can never observe the
# live source.
@@ -1029,73 +1311,46 @@ class ResourceService:
**kwargs,
)
- if git_repo_source:
- if "auth_config" in kwargs:
- git_auth = parse_git_http_auth_config(
- kwargs["auth_config"],
- path,
- )
- if git_auth is None:
- raise InvalidArgumentError("args.auth_config must be an object.")
-
- watch_auth_state = normalized_args.watch_auth_state
- if watch_interval > 0:
- watch_auth_state = create_git_http_auth_state(git_auth, path)
-
- # The native Git queue is durable, so credentials must be consumed
- # before crossing that boundary. Fetch and parse in this request;
- # _execute_resource_ingestion only queues the credential-free
- # prepared post-processing payload when defer_post_processing=True.
- request_local_kwargs = dict(kwargs)
- request_local_kwargs["auth_config"] = {
- "username": git_auth.username,
- "token": git_auth.token,
- }
- result = await self._execute_resource_ingestion(
- path=path,
- ctx=ctx,
- to=to,
- to_is_directory=to_is_directory,
- parent=parent,
- reason=reason,
- instruction=instruction,
- defer_post_processing=True,
- timeout=timeout,
- build_index=build_index,
- summarize=summarize,
- processing_mode=processing_mode,
- parse_mode=mode,
- watch_interval=watch_interval,
- manage_watch=manage_watch,
- tags=tags,
- tag_mode=tag_mode,
- allow_local_path_resolution=allow_local_path_resolution,
- enforce_public_remote_targets=enforce_public_remote_targets,
- watch_auth_state=watch_auth_state,
- **request_local_kwargs,
- )
- else:
- result = await self.enqueue_git_add_resource(
- path=path,
- ctx=ctx,
- to=to,
- to_is_directory=to_is_directory,
- parent=parent,
- reason=reason,
- instruction=instruction,
- timeout=timeout,
- build_index=build_index,
- summarize=summarize,
- processing_mode=processing_mode,
- parse_mode=mode,
- watch_interval=watch_interval,
- manage_watch=manage_watch,
- tags=tags,
- tag_mode=tag_mode,
- allow_local_path_resolution=allow_local_path_resolution,
- enforce_public_remote_targets=enforce_public_remote_targets,
- **kwargs,
- )
+ if enforce_public_remote_targets and is_remote_resource_source(path):
+ path = require_remote_resource_source(path)
+ kwargs.setdefault("request_validator", ensure_public_remote_target)
+
+ target = ContentTargetSpec.from_fields(
+ ctx=ctx,
+ kind="resource",
+ to=to,
+ parent=parent,
+ create_parent=bool(kwargs.get("create_parent", False)),
+ )
+ source_plan = await self._prepare_standard_source_plan(
+ path=path,
+ ctx=ctx,
+ mode=mode,
+ allow_local_path_resolution=allow_local_path_resolution,
+ processor_kwargs=kwargs,
+ watch_auth_state=normalized_args.watch_auth_state,
+ )
+ if source_plan is not None:
+ result = await self._enqueue_source_plan(
+ source_plan,
+ ctx=ctx,
+ target=target,
+ reason=reason,
+ instruction=instruction,
+ timeout=timeout,
+ build_index=build_index,
+ summarize=summarize,
+ processing_mode=processing_mode,
+ mode=mode,
+ watch_interval=watch_interval,
+ manage_watch=manage_watch,
+ tags=tags,
+ tag_mode=tag_mode,
+ to_is_directory=to_is_directory,
+ allow_local_path_resolution=allow_local_path_resolution,
+ enforce_public_remote_targets=enforce_public_remote_targets,
+ processor_kwargs=kwargs,
+ )
else:
result = await self._execute_resource_ingestion(
path=path,
@@ -1118,7 +1373,6 @@ class ResourceService:
allow_local_path_resolution=allow_local_path_resolution,
enforce_public_remote_targets=enforce_public_remote_targets,
watch_auth_state=normalized_args.watch_auth_state,
- parser_args=normalized_args.processor_kwargs,
**kwargs,
)
get_current_telemetry().set("resource.flags.wait", wait)
@@ -1172,9 +1426,9 @@ class ResourceService:
allow_local_path_resolution: bool = True,
enforce_public_remote_targets: bool = False,
watch_auth_state: Optional[Dict[str, Any]] = None,
- parser_args: Optional[Dict[str, Any]] = None,
resource_lock: Optional[Dict[str, Any]] = None,
stage_callback: Optional[Callable[[str], Any]] = None,
+ prepared_resource: Optional["LocalResource"] = None,
**kwargs,
) -> Dict[str, Any]:
"""Execute an already-routed resource ingestion."""
@@ -1199,7 +1453,6 @@ class ResourceService:
telemetry.set("resource.flags.summarize", summarize)
telemetry.set("resource.flags.watch_enabled", watch_enabled)
- prepared_resource: Optional["LocalResource"] = None
try:
target = ContentTargetSpec.from_fields(
ctx=ctx,
@@ -1217,209 +1470,6 @@ class ResourceService:
if resource_lock is not None:
kwargs["resource_lock"] = resource_lock
- async_understanding_candidate = (
- mode is ParseMode.DEFAULT
- and defer_post_processing
- and not is_git_repo_url(path)
- and not allow_local_path_resolution
- and self._resource_processor is not None
- )
- direct_understanding = bool(
- async_understanding_candidate
- and self._resource_processor.should_use_understanding_directly(path, **kwargs)
- )
-
- if (
- async_understanding_candidate
- and not direct_understanding
- and self._resource_processor.understanding_api_enabled()
- ):
- prepared_resource = await self._resource_processor.prepare_resource(
- path,
- ctx,
- allow_local_path_resolution=allow_local_path_resolution,
- **kwargs,
- )
-
- if (
- prepared_resource is not None
- and self._resource_processor is not None
- and self._resource_processor.should_use_understanding_api(prepared_resource)
- ):
- understanding_source = prepared_resource
- elif direct_understanding:
- understanding_source = path
- else:
- understanding_source = None
-
- if understanding_source is not None:
- from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
-
- if prepared_resource is not None:
- resolved_extension = str(prepared_resource.meta.get("resolved_extension") or "")
- source_name = (
- kwargs.get("source_name")
- or prepared_resource.meta.get("original_filename")
- or prepared_resource.meta.get("resolved_name")
- )
- source_format = resolved_extension.lstrip(".") or "file"
- if resolved_extension.lower().lstrip(".") == MPEG_TS_EXTENSION_ALIAS:
- source_format = "video"
- source_info = _ResourceSourceInfo(
- source_name=source_name,
- source_path=path,
- source_format=source_format,
- )
- else:
- resolved_extension = ""
- source_name = kwargs.get("source_name")
- source_info = _ResourceSourceInfo(
- source_name=source_name,
- source_path=path,
- source_format=resolved_extension.lstrip(".") or "file",
- )
- doc_name = self._target_doc_name(path, source_name, source_info)
- source_path = source_info.source_path or source_name or path
- (
- root_uri,
- candidate_uri,
- ) = await self._resource_processor.tree_builder.resolve_target_uri(
- ctx=ctx,
- doc_name=doc_name,
- scope="resources",
- to_uri=target.to,
- parent_uri=target.parent,
- source_path=source_path,
- source_format=source_info.source_format,
- create_parent=target.create_parent,
- )
- defer_target_resolution = bool(
- candidate_uri and not source_name and not watch_enabled and direct_understanding
- )
- if self._viking_fs is None:
- raise NotInitializedError("VikingFS")
- from openviking.storage.errors import ResourceBusyError
-
- lock_lease: Optional[Dict[str, Any]] = None
-
- async def _reserve_tree(uri: str) -> Dict[str, Any]:
- dst_path = self._viking_fs._uri_to_path(uri, ctx=ctx)
- try:
- return await self._viking_fs._async_agfs.pathlock_acquire_tree(
- dst_path,
- timeout_secs=0.0,
- )
- except Exception as exc:
- raise ResourceBusyError(
- f"Resource is busy: {uri}",
- uri=uri,
- conflict_type="path_busy",
- retryable=True,
- ) from exc
-
- if candidate_uri and not defer_target_resolution:
- root_uri, lock_lease = await self._resource_processor.reserve_unique_candidate(
- candidate_uri=candidate_uri,
- ctx=ctx,
- )
- elif not defer_target_resolution:
- lock_lease = await _reserve_tree(root_uri)
-
- enqueue_started = False
- try:
- queued_args = dict(parser_args or {})
- understanding_response_id = await self._resource_processor.submit_understanding(
- understanding_source,
- **queued_args,
- )
- queued_args.pop(FEISHU_ACCESS_TOKEN_ARG, None)
- if prepared_resource is not None:
- prepared_resource.cleanup()
- prepared_resource = None
-
- processor_args = self._sanitize_watch_processor_kwargs(queued_args)
- processor_args["parser_backend"] = "understanding"
- if resolved_extension:
- processor_args["resolved_extension"] = resolved_extension
-
- lock_handoff = await self._lock_to_handoff_payload(lock_lease)
- msg = AddResourceMsg(
- task_id=str(uuid4()),
- telemetry_id=telemetry_id or None,
- path=path,
- source_path=(source_name or "") if kwargs.get("temp_file_id") else path,
- root_uri=root_uri,
- account_id=ctx.account_id,
- user_id=ctx.user.user_id,
- role=str(ctx.role),
- actor_peer_id=ctx.actor_peer_id,
- reason=reason,
- instruction=instruction,
- timeout=timeout,
- build_index=build_index,
- summarize=summarize,
- processing_mode=processing_mode,
- strict=bool(kwargs.get("strict", False)),
- ignore_dirs=kwargs.get("ignore_dirs"),
- include=kwargs.get("include"),
- exclude=kwargs.get("exclude"),
- directly_upload_media=bool(kwargs.get("directly_upload_media", True)),
- preserve_structure=kwargs.get("preserve_structure"),
- create_parent=bool(kwargs.get("create_parent", False)),
- allow_local_path_resolution=allow_local_path_resolution,
- enforce_public_remote_targets=enforce_public_remote_targets,
- args=processor_args,
- source_name=source_name,
- lock_handoff=lock_handoff,
- skip_watch_management=True,
- defer_target_resolution=defer_target_resolution,
- understanding_response_id=understanding_response_id,
- tags=tags,
- tag_mode=tag_mode,
- )
- enqueue_started = True
- enqueue_lock = lock_lease
- lock_lease = None
- task = await self._enqueue_add_resource_job(
- msg,
- queue_name=QueueManager.EXTERNAL_PARSE,
- resource_lock=enqueue_lock,
- )
- except BaseException:
- if not enqueue_started and lock_lease is not None:
- await self._release_lock_ref(lock_lease)
- raise
- job_enqueued = True
- logger.info(
- "[ResourceService] Enqueued AddResourceMsg task_id=%s root_uri=%s",
- task.task_id,
- root_uri,
- )
- await self._manage_watch_if_needed(
- watch_manager=watch_manager,
- manage_watch=manage_watch,
- watch_interval=watch_interval,
- target=target,
- to_is_directory=watch_to_is_directory,
- root_uri=root_uri,
- path=path,
- reason=reason,
- instruction=instruction,
- build_index=build_index,
- summarize=summarize,
- processing_mode=processing_mode,
- processor_kwargs=self._watch_processor_kwargs(kwargs, tags, tag_mode),
- watch_auth_state=watch_auth_state,
- ctx=ctx,
- )
- response = {
- "status": "success",
- "task_id": task.task_id,
- }
- if not defer_target_resolution:
- response["root_uri"] = root_uri
- return response
-
result = await self._resource_processor.process_resource(
path=path,
ctx=ctx,
@@ -1435,7 +1485,7 @@ class ResourceService:
stage_callback=stage_callback,
allow_local_path_resolution=allow_local_path_resolution,
prepared_resource=prepared_resource,
- defer_post_processing=defer_post_processing,
+ defer_post_processing=True,
**ingest_tag_kwargs,
**kwargs,
)
@@ -1446,20 +1496,38 @@ class ResourceService:
prepared = result.pop("_post_process", None)
deferred_lock = result.pop("_resource_lock", None)
if (
- not target.to
+ not to_is_directory
and isinstance(prepared, dict)
and isinstance(prepared.get("root_is_file"), bool)
):
watch_to_is_directory = not prepared["root_is_file"]
+ if not isinstance(prepared, dict):
+ raise InternalError("Deferred resource processing payload is missing")
+ await self._manage_watch_if_needed(
+ watch_manager=watch_manager,
+ manage_watch=manage_watch,
+ watch_interval=watch_interval,
+ target=target,
+ to_is_directory=watch_to_is_directory,
+ root_uri=str(result.get("root_uri") or ""),
+ path=path,
+ reason=reason,
+ instruction=instruction,
+ build_index=build_index,
+ summarize=summarize,
+ processing_mode=processing_mode,
+ processor_kwargs=self._watch_processor_kwargs(kwargs, tags, tag_mode),
+ watch_auth_state=watch_auth_state,
+ ctx=ctx,
+ )
if defer_post_processing:
from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
root_uri = result.get("root_uri", "")
- if not isinstance(prepared, dict):
- raise InternalError("Deferred resource processing payload is missing")
lock_handoff = await self._lock_to_handoff_payload(deferred_lock)
msg = AddResourceMsg(
task_id=str(uuid4()),
+ job_phase=AddResourcePhase.POST_PROCESS,
root_uri=root_uri,
prepared=prepared,
source_path=str(
@@ -1502,23 +1570,32 @@ class ResourceService:
)
result["task_id"] = task.task_id
job_enqueued = True
- await self._manage_watch_if_needed(
- watch_manager=watch_manager,
- manage_watch=manage_watch,
- watch_interval=watch_interval,
- target=target,
- to_is_directory=watch_to_is_directory,
- root_uri=str(result.get("root_uri") or ""),
- path=path,
- reason=reason,
- instruction=instruction,
- build_index=build_index,
- summarize=summarize,
- processing_mode=processing_mode,
- processor_kwargs=self._watch_processor_kwargs(kwargs, tags, tag_mode),
- watch_auth_state=watch_auth_state,
- ctx=ctx,
- )
+ else:
+ processing_lock = deferred_lock
+ deferred_lock = None
+ post_process_kwargs = dict(kwargs)
+ for key in (
+ "resource_lock",
+ "request_validator",
+ "auth_config",
+ FEISHU_ACCESS_TOKEN_ARG,
+ FEISHU_REFRESH_TOKEN_ARG,
+ "parser_backend",
+ "resolved_extension",
+ ):
+ post_process_kwargs.pop(key, None)
+ post_result = await self._resource_processor.finish_prepared_resource(
+ prepared,
+ ctx=ctx,
+ resource_lock=processing_lock,
+ summarize=summarize,
+ build_index=build_index,
+ processing_mode=processing_mode,
+ **ingest_tag_kwargs,
+ **post_process_kwargs,
+ )
+ if post_result.get("warnings"):
+ result.setdefault("warnings", []).extend(post_result["warnings"])
return result
except Exception as exc:
telemetry.set_error(
diff --git a/openviking/service/task_store.py b/openviking/service/task_store.py
index 741c81aad..1e122c93a 100644
--- a/openviking/service/task_store.py
+++ b/openviking/service/task_store.py
@@ -154,6 +154,7 @@ def _task_to_payload(task: Any) -> Dict[str, Any]:
"stage": task.stage,
"result": deepcopy(task.result),
"error": task.error,
+ "auth": deepcopy(task.auth),
}
diff --git a/openviking/service/task_tracker.py b/openviking/service/task_tracker.py
index e18ab92cb..04874001b 100644
--- a/openviking/service/task_tracker.py
+++ b/openviking/service/task_tracker.py
@@ -80,6 +80,7 @@ class TaskRecord:
stage: Optional[str] = None
result: Optional[Dict[str, Any]] = None
error: Optional[str] = None
+ auth: Dict[str, Any] = field(default_factory=dict, repr=False)
def to_dict(self) -> Dict[str, Any]:
"""Serialize for JSON response."""
@@ -89,6 +90,7 @@ class TaskRecord:
d["updated_at_iso"] = datetime.fromtimestamp(self.updated_at, tz=timezone.utc).isoformat()
d["result"] = _sanitize_task_result(d.get("result"))
d["meta"] = _sanitize_task_result(d.get("meta"))
+ d.pop("auth", None)
d.pop("account_id", None)
d.pop("user_id", None)
return d
@@ -353,6 +355,7 @@ class TaskTracker:
user_id: str,
task_id: Optional[str] = None,
meta: Optional[Dict[str, Any]] = None,
+ auth: Optional[Dict[str, Any]] = None,
) -> TaskRecord:
"""Register a new pending task. Returns a snapshot copy."""
self._validate_owner(account_id, user_id)
@@ -363,6 +366,7 @@ class TaskTracker:
account_id=account_id,
user_id=user_id,
meta=dict(meta or {}),
+ auth=dict(auth or {}),
)
return await self._dispatcher.run(lambda: self._create_on_owner(task, task_id is not None))
@@ -587,6 +591,7 @@ class TaskTracker:
if work_error and updated.error is None:
updated.error = _sanitize_error(work_error)
updated.updated_at = self._next_updated_at(task)
+ updated.auth = {}
try:
await self._persist_and_publish("update", updated)
except _CommittedMutationCancelled as exc:
@@ -713,6 +718,7 @@ class TaskTracker:
return
updated.stage = updated.status.value
updated.updated_at = self._next_updated_at(task)
+ updated.auth = {}
await self._persist_and_publish("update", updated)
self._work_index.clear_failure(task_id)
logger.info("[TaskTracker] Task %s %s", task_id, updated.status.value)
@@ -740,6 +746,28 @@ class TaskTracker:
return await _poll()
return await asyncio.wait_for(_poll(), timeout)
+ async def get_task_auth(
+ self,
+ task_id: str,
+ *,
+ account_id: str,
+ user_id: str,
+ ) -> Dict[str, Any]:
+ """Load task-owned authentication excluded from public snapshots."""
+ self._validate_owner(account_id, user_id)
+
+ async def load() -> Dict[str, Any]:
+ task = self._cached_task(task_id)
+ if task is None:
+ task = await self._load_from_store(task_id, account_id, user_id)
+ if task is not None:
+ self._publish_task(task)
+ if task is None or not self._matches_owner(task, account_id, user_id):
+ return {}
+ return deepcopy(task.auth)
+
+ return await self._dispatcher.run(load)
+
async def delete_user_tasks(self, account_id: str, user_id: str) -> int:
"""Delete terminal task records for one user from storage and cache."""
self._validate_owner(account_id, user_id)
@@ -1021,6 +1049,7 @@ class TaskTracker:
copied = deepcopy(task)
copied.meta = _sanitize_task_result(copied.meta)
copied.result = _sanitize_task_result(copied.result)
+ copied.auth = {}
return copied
def count(self) -> int:
diff --git a/openviking/storage/queuefs/add_resource_msg.py b/openviking/storage/queuefs/add_resource_msg.py
index bfbc187a7..ae0cb7461 100644
--- a/openviking/storage/queuefs/add_resource_msg.py
+++ b/openviking/storage/queuefs/add_resource_msg.py
@@ -1,13 +1,19 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
-"""Persistent add-resource queue message (legacy ExternalParse queue payload)."""
+"""Persistent messages for the resource source and post-process job phases."""
from dataclasses import asdict, dataclass, field
+from enum import Enum
from typing import Any, Dict, Optional
from openviking.resource.processing_mode import DEFAULT_PROCESSING_MODE, ProcessingMode
+class AddResourcePhase(str, Enum):
+ SOURCE = "source"
+ POST_PROCESS = "post_process"
+
+
@dataclass(kw_only=True)
class AddResourceMsg:
task_id: str
@@ -19,6 +25,8 @@ class AddResourceMsg:
source_path: str = ""
telemetry_id: Optional[str] = None
prepared: Optional[Dict[str, Any]] = None
+ staged_source: Optional[Dict[str, Any]] = None
+ job_phase: AddResourcePhase | str | None = None
lock_handoff: Optional[Dict[str, Any]] = None
actor_peer_id: Optional[str] = None
reason: str = ""
@@ -38,6 +46,7 @@ class AddResourceMsg:
args: Dict[str, Any] = field(default_factory=dict)
lock_handoff_retry: int = 0
source_name: Optional[str] = None
+ to_is_directory: Optional[bool] = None
watch_interval: float = 0
skip_watch_management: bool = True
defer_target_resolution: bool = False
@@ -47,6 +56,26 @@ class AddResourceMsg:
tags: Optional[list[str]] = None
tag_mode: str = "replace"
+ def __post_init__(self) -> None:
+ inferred = (
+ AddResourcePhase.POST_PROCESS if self.prepared is not None else AddResourcePhase.SOURCE
+ )
+ try:
+ phase = AddResourcePhase(self.job_phase or inferred)
+ except ValueError as exc:
+ raise ValueError(f"Unsupported add-resource job phase: {self.job_phase}") from exc
+ if phase is AddResourcePhase.POST_PROCESS and self.prepared is None:
+ raise ValueError("post_process jobs require prepared data")
+ if phase is AddResourcePhase.SOURCE and self.prepared is not None:
+ raise ValueError("source jobs cannot contain prepared post-process data")
+ if self.prepared is not None and (
+ self.staged_source is not None or self.understanding_response_id is not None
+ ):
+ raise ValueError("post_process jobs cannot contain source payloads")
+ if self.staged_source is not None and self.understanding_response_id is not None:
+ raise ValueError("source jobs cannot contain multiple source payloads")
+ self.job_phase = phase
+
def to_dict(self) -> Dict[str, Any]:
data = asdict(self)
if self.prepared is not None:
@@ -61,6 +90,18 @@ class AddResourceMsg:
path = data.get("path")
root_uri = data.get("root_uri")
prepared = data.get("prepared") if isinstance(data.get("prepared"), dict) else None
+ staged_source = None
+ if data.get("staged_source") is not None:
+ from openviking.resource.staged_source import StagedSource
+
+ staged_source = StagedSource.from_dict(data["staged_source"]).to_dict()
+ if prepared is not None and staged_source is not None:
+ raise ValueError("prepared and staged_source are mutually exclusive")
+ job_phase = data.get("job_phase") or (
+ AddResourcePhase.POST_PROCESS.value
+ if prepared is not None
+ else AddResourcePhase.SOURCE.value
+ )
args = dict(data.get("args", {})) if isinstance(data.get("args"), dict) else {}
legacy_retry = args.pop("_lock_handoff_retry", 0)
try:
@@ -69,12 +110,12 @@ class AddResourceMsg:
lock_handoff_retry = 0
if prepared is not None:
args.clear()
- if not task_id or (not path and not prepared) or not root_uri:
+ if not task_id or (not path and not prepared and not staged_source) or not root_uri:
missing = []
if not task_id:
missing.append("task_id")
- if not path and not prepared:
- missing.append("path or prepared")
+ if not path and not prepared and not staged_source:
+ missing.append("path, prepared, or staged_source")
if not root_uri:
missing.append("root_uri")
raise ValueError(f"Missing required fields: {missing}")
@@ -115,7 +156,12 @@ class AddResourceMsg:
args=args,
lock_handoff_retry=lock_handoff_retry,
source_name=data.get("source_name"),
+ to_is_directory=(
+ bool(data["to_is_directory"]) if data.get("to_is_directory") is not None else None
+ ),
prepared=prepared,
+ staged_source=staged_source,
+ job_phase=job_phase,
watch_interval=float(data.get("watch_interval", 0) or 0),
skip_watch_management=bool(data.get("skip_watch_management", True)),
defer_target_resolution=bool(data.get("defer_target_resolution", False)),
@@ -126,10 +172,6 @@ class AddResourceMsg:
),
processing_mode=data.get("processing_mode", DEFAULT_PROCESSING_MODE),
parse_mode=str(data.get("parse_mode") or "default"),
- tags=(
- list(data["tags"])
- if isinstance(data.get("tags"), list)
- else None
- ),
+ tags=(list(data["tags"]) if isinstance(data.get("tags"), list) else None),
tag_mode=str(data.get("tag_mode") or "replace"),
)
diff --git a/openviking/storage/queuefs/add_resource_processor.py b/openviking/storage/queuefs/add_resource_processor.py
index 5aac40cce..3ea345ce8 100644
--- a/openviking/storage/queuefs/add_resource_processor.py
+++ b/openviking/storage/queuefs/add_resource_processor.py
@@ -54,14 +54,27 @@ class AddResourceProcessor(DequeueHandlerBase):
except Exception:
raise handoff_error
- async def _release_cancelled_handoff(self, msg: AddResourceMsg) -> None:
- if msg.lock_handoff is None:
+ async def _cleanup_staged_source(self, msg: AddResourceMsg, ctx: RequestContext) -> None:
+ if msg.staged_source is None:
return
- try:
- lock = await self._viking_fs._async_agfs.pathlock_adopt(msg.lock_handoff)
- await self._viking_fs._async_agfs.pathlock_release(lock)
- except Exception as exc:
- logger.warning("[AddResource] Failed to release cancelled lock handoff: %s", exc)
+ from openviking.resource.staged_source import StagedSource
+
+ staged = StagedSource.from_dict(msg.staged_source)
+ await self._viking_fs.delete_temp(staged.temp_uri, ctx=ctx)
+
+ async def _release_cancelled_resources(
+ self,
+ msg: AddResourceMsg,
+ ctx: RequestContext,
+ ) -> None:
+ if msg.lock_handoff is not None:
+ try:
+ lock = await self._viking_fs._async_agfs.pathlock_adopt(msg.lock_handoff)
+ await self._viking_fs._async_agfs.pathlock_release(lock)
+ except Exception as exc:
+ logger.warning("[AddResource] Failed to release cancelled lock handoff: %s", exc)
+ with suppress(Exception):
+ await self._cleanup_staged_source(msg, ctx)
async def _requeue_lock_handoff(self, msg: AddResourceMsg, exc: Exception) -> bool:
if msg.lock_handoff_retry >= 2:
@@ -104,7 +117,10 @@ class AddResourceProcessor(DequeueHandlerBase):
TaskStatus.CANCELLED,
):
if task.status in (TaskStatus.CANCELLING, TaskStatus.CANCELLED):
- await self._release_cancelled_handoff(msg)
+ await self._release_cancelled_resources(msg, ctx)
+ else:
+ with suppress(Exception):
+ await self._cleanup_staged_source(msg, ctx)
unregister_telemetry(telemetry_id)
self.report_success()
return None
@@ -126,6 +142,8 @@ class AddResourceProcessor(DequeueHandlerBase):
)
self.report_error(f"Invalid lock_handoff: {exc}", data)
unregister_telemetry(telemetry_id)
+ with suppress(Exception):
+ await self._cleanup_staged_source(msg, ctx)
return None
telemetry = resolve_telemetry(telemetry_id) if telemetry_id else None
@@ -151,6 +169,7 @@ class AddResourceProcessor(DequeueHandlerBase):
bind_telemetry(telemetry),
bind_task_context(msg.task_id, ctx.account_id, ctx.user.user_id),
):
+ terminal = False
try:
if replay_result is None:
await tracker.start(
@@ -164,6 +183,11 @@ class AddResourceProcessor(DequeueHandlerBase):
ctx=ctx,
resource_lock=resource_lock,
stage_callback=_set_stage,
+ task_auth=await tracker.get_task_auth(
+ msg.task_id,
+ account_id=ctx.account_id,
+ user_id=ctx.user.user_id,
+ ),
)
if result.get("status") == "error":
errors = result.get("errors") or ["resource processing failed"]
@@ -173,6 +197,7 @@ class AddResourceProcessor(DequeueHandlerBase):
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
+ terminal = True
self.report_error("resource processing failed", data)
return None
await tracker.complete(
@@ -217,6 +242,7 @@ class AddResourceProcessor(DequeueHandlerBase):
user_id=ctx.user.user_id,
resource_id=result.get("root_uri"),
)
+ terminal = True
self.report_success()
return None
except Exception as exc:
@@ -226,6 +252,7 @@ class AddResourceProcessor(DequeueHandlerBase):
account_id=ctx.account_id,
user_id=ctx.user.user_id,
)
+ terminal = True
self.report_error(str(exc), data)
return None
finally:
@@ -234,6 +261,9 @@ class AddResourceProcessor(DequeueHandlerBase):
with suppress(Exception):
if resource_lock is not None:
await self._viking_fs._async_agfs.pathlock_release(resource_lock)
+ if terminal:
+ with suppress(Exception):
+ await self._cleanup_staged_source(msg, ctx)
async def on_cancelled(self, data: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
"""Release an enqueue-time lock before ACKing cancelled work."""
@@ -246,7 +276,14 @@ class AddResourceProcessor(DequeueHandlerBase):
self.report_error(str(exc), data)
return None
future = asyncio.run_coroutine_threadsafe(
- self._release_cancelled_handoff(msg),
+ self._release_cancelled_resources(
+ msg,
+ RequestContext(
+ user=UserIdentifier(msg.account_id, msg.user_id),
+ role=Role(msg.role),
+ actor_peer_id=msg.actor_peer_id,
+ ),
+ ),
self._service_loop,
)
await asyncio.wrap_future(future)
diff --git a/openviking/utils/git_auth.py b/openviking/utils/git_auth.py
index c3d47074c..0d8403fc4 100644
--- a/openviking/utils/git_auth.py
+++ b/openviking/utils/git_auth.py
@@ -8,6 +8,7 @@ import base64
import os
from dataclasses import dataclass, field
from typing import Mapping
+from urllib.parse import urlparse
from openviking_cli.exceptions import InvalidArgumentError
@@ -20,6 +21,17 @@ class GitHttpAuthConfig:
token: str = field(repr=False)
+def reject_git_http_userinfo(repo_url: str) -> None:
+ """Require HTTP(S) Git credentials to travel through ``auth_config``."""
+ parsed = urlparse(repo_url)
+ if parsed.scheme.lower() in {"http", "https"} and (
+ parsed.username is not None or parsed.password is not None
+ ):
+ raise InvalidArgumentError(
+ "HTTP(S) Git URLs cannot contain userinfo; use args.auth_config instead."
+ )
+
+
def _validate_git_https_auth_url(repo_url: str) -> str:
normalized = repo_url.strip() if isinstance(repo_url, str) else ""
if not normalized.lower().startswith("https://"):
diff --git a/openviking/utils/media_processor.py b/openviking/utils/media_processor.py
index e121005b4..589a3ba19 100644
--- a/openviking/utils/media_processor.py
+++ b/openviking/utils/media_processor.py
@@ -4,9 +4,11 @@
from pathlib import Path
from typing import TYPE_CHECKING, Optional
+from urllib.parse import urlparse
from openviking.parse.accessors.base import LocalResource, SourceType
from openviking.parse.accessors.mime_types import IANA_MEDIA_TYPE_TO_EXTENSION
+from openviking.parse.backend import ParserBackend, normalize_parser_backend
from openviking.parse.base import ParseResult
from openviking.parse.mode import ParseMode, normalize_parse_mode
from openviking.parse.parsers.constants import (
@@ -123,6 +125,36 @@ class UnifiedResourceProcessor:
def should_use_understanding_directly(self, source: str, **kwargs) -> bool:
return self._get_parser_router().should_use_understanding_directly(source, **kwargs)
+ def durable_route_requires_preparation(
+ self,
+ source: str,
+ *,
+ parse_mode: ParseMode | str = ParseMode.DEFAULT,
+ **kwargs,
+ ) -> bool:
+ """Return whether durable parser selection must inspect fetched content."""
+ if normalize_parse_mode(parse_mode) is ParseMode.NO_SPLIT:
+ return False
+ parser_backend = normalize_parser_backend(kwargs.get("parser_backend"))
+ if parser_backend is ParserBackend.INTERNAL:
+ return False
+
+ from openviking.parse.accessors.feishu_accessor import FeishuAccessor
+ from openviking.parse.accessors.web_feed_accessor import WebFeedAccessor
+
+ accessor = self._get_accessor_registry().get_accessor(source, **kwargs)
+ if isinstance(accessor, (FeishuAccessor, WebFeedAccessor)):
+ return False
+
+ router = self._get_parser_router()
+ if not router.understanding_api_enabled():
+ return False
+ if parser_backend is ParserBackend.UNDERSTANDING:
+ return True
+ if not Path(urlparse(source).path).suffix:
+ return True
+ return router.should_use_understanding_api(source)
+
async def submit_understanding(self, source: str | Path | LocalResource, **kwargs) -> str:
return await self._get_parser_router().submit(source, **kwargs)
@@ -216,7 +248,7 @@ class UnifiedResourceProcessor:
"original_url": source,
}
parse_kwargs["original_source"] = source
- parse_kwargs["parser_backend"] = "understanding"
+ parse_kwargs["parser_backend"] = ParserBackend.UNDERSTANDING
parse_kwargs.pop("feishu_access_token", None)
explicit_name = kwargs.get("resource_name") or kwargs.get("source_name")
diff --git a/openviking/utils/resource_processor.py b/openviking/utils/resource_processor.py
index bdb9e8198..6f498eb16 100644
--- a/openviking/utils/resource_processor.py
+++ b/openviking/utils/resource_processor.py
@@ -103,17 +103,24 @@ class ResourceProcessor:
)
return self._media_processor
- async def prepare_resource(
+ async def prepare_durable_source(
self,
path: str,
ctx: RequestContext,
*,
+ snapshot_required: bool = False,
allow_local_path_resolution: bool = True,
**kwargs,
- ) -> "LocalResource":
- """Resolve a source once before selecting an async parser route."""
+ ) -> Optional["LocalResource"]:
+ """Freeze a source when durable routing cannot safely defer access."""
+ media_processor = self._get_media_processor()
+ if (
+ not snapshot_required
+ and not media_processor.durable_route_requires_preparation(path, **kwargs)
+ ):
+ return None
with get_viking_fs().bind_request_context(ctx):
- return await self._get_media_processor().prepare(
+ return await media_processor.prepare(
path,
allow_local_path_resolution=allow_local_path_resolution,
**kwargs,
diff --git a/tests/parse/test_feishu_accessor.py b/tests/parse/test_feishu_accessor.py
index b022aba57..033f8c2a7 100644
--- a/tests/parse/test_feishu_accessor.py
+++ b/tests/parse/test_feishu_accessor.py
@@ -621,6 +621,46 @@ def test_fetch_document_dispatches_all_supported_types(monkeypatch):
)
assert handlers["_parse_bitable"].call_args_list[-1].args == ("wiki_app_token", None)
+ monkeypatch.setattr(accessor, "_probe_docx_document", MagicMock())
+ monkeypatch.setattr(
+ accessor,
+ "_fetch_spreadsheet_metadata",
+ MagicMock(return_value={"properties": {"title": "Sheet/Title"}, "sheets": []}),
+ )
+ monkeypatch.setattr(
+ accessor,
+ "_list_bitable_tables",
+ MagicMock(
+ return_value=[
+ SimpleNamespace(table_id="tbl_one", name="One"),
+ SimpleNamespace(table_id="tbl_two", name="Two"),
+ ]
+ ),
+ )
+ monkeypatch.setattr(accessor, "_probe_bitable_table", MagicMock())
+ monkeypatch.setattr(accessor, "_get_drive_folder_name", MagicMock(return_value="Docs/Root"))
+ monkeypatch.setattr(accessor, "_probe_drive_folder_children", MagicMock())
+
+ docx_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/docx/doc"))
+ sheets_identity = asyncio.run(
+ accessor.preflight_source("https://example.feishu.cn/sheets/sht")
+ )
+ base_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/base/app"))
+ table_identity = asyncio.run(
+ accessor.preflight_source("https://example.feishu.cn/base/app?table=tbl_one")
+ )
+ folder_identity = asyncio.run(
+ accessor.preflight_source("https://example.feishu.cn/drive/folder/fld")
+ )
+ wiki_identity = asyncio.run(accessor.preflight_source("https://example.feishu.cn/wiki/wiki"))
+
+ assert docx_identity.source_name is None
+ assert sheets_identity.source_name == "Sheet_Title"
+ assert base_identity.source_name == "Bitable (2 tables)"
+ assert table_identity.source_name == "tbl_one"
+ assert folder_identity.source_name == "Docs_Root"
+ assert wiki_identity.source_name == "Wiki Base"
+
def test_fetch_document_honors_bitable_table_and_view(monkeypatch):
_install_fake_lark_modules(monkeypatch)
@@ -668,6 +708,21 @@ def test_fetch_document_honors_bitable_table_and_view(monkeypatch):
assert document.meta["feishu_table_id"] == "tblSales"
assert document.meta["feishu_view_id"] == "vewPublic"
+ list_fields.reset_mock()
+ list_records.reset_mock()
+ identity = asyncio.run(
+ accessor.preflight_source(
+ "https://example.feishu.cn/base/app_token?table=tblSales&view=vewPublic"
+ )
+ )
+
+ assert identity.source_name == "tblSales (vewPublic)"
+ assert list_tables.call_count == 0
+ assert list_fields.call_count == 1
+ assert list_records.call_count == 1
+ request = list_records.call_args.args[0]
+ assert (request.table_id, request.view_id) == ("tblSales", "vewPublic")
+
def test_fetch_document_rejects_bitable_view_without_table():
accessor = FeishuAccessor()
diff --git a/tests/parse/test_feishu_errors.py b/tests/parse/test_feishu_errors.py
index 9cae0cb78..d30f351e4 100644
--- a/tests/parse/test_feishu_errors.py
+++ b/tests/parse/test_feishu_errors.py
@@ -29,9 +29,10 @@ def _raised_error(response, *, operation="resolve wiki node", resource=None):
return exc_info.value
-def test_maps_feishu_forbidden_to_permission_denied_and_keeps_details():
[email protected]("feishu_code", [1770032, 131006])
+def test_maps_feishu_forbidden_to_permission_denied_and_keeps_details(feishu_code):
response = _fake_response(
- code=1770032,
+ code=feishu_code,
msg="forBidden",
http_status=400,
)
@@ -42,8 +43,8 @@ def test_maps_feishu_forbidden_to_permission_denied_and_keeps_details():
)
assert exc.code == "PERMISSION_DENIED"
- assert "code=1770032, msg=forBidden" in exc.message
- assert exc.details["feishu_code"] == 1770032
+ assert f"code={feishu_code}, msg=forBidden" in exc.message
+ assert exc.details["feishu_code"] == feishu_code
assert exc.details["feishu_msg"] == "forBidden"
assert exc.details["http_status"] == 400
assert exc.details["resource"] == "doc_token"
diff --git a/tests/parse/test_feishu_parser_api.py b/tests/parse/test_feishu_parser_api.py
index a15774214..15c57daaf 100644
--- a/tests/parse/test_feishu_parser_api.py
+++ b/tests/parse/test_feishu_parser_api.py
@@ -13,7 +13,7 @@ from openviking.server.identity import RequestContext, Role
from openviking.service.resource_service import ResourceService
from openviking.service.task_tracker import TaskStatus
from openviking.service.task_work_index import TASK_WORK_ID_FIELD
-from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
+from openviking.storage.queuefs.add_resource_msg import AddResourceMsg, AddResourcePhase
from openviking.storage.queuefs.add_resource_processor import AddResourceProcessor
from openviking.storage.queuefs.queue_manager import QueueManager
from openviking.utils.media_processor import UnifiedResourceProcessor
@@ -389,6 +389,7 @@ def test_add_resource_message_round_trips_internal_fields():
assert "feishu_access_token" not in json.dumps(restored.to_dict())
assert restored.defer_target_resolution is True
assert restored.understanding_response_id == "response-1"
+ assert restored.job_phase is AddResourcePhase.SOURCE
def test_add_resource_message_round_trips_processing_mode():
@@ -408,16 +409,24 @@ def test_add_resource_message_round_trips_processing_mode():
@pytest.mark.asyncio
-async def test_uat_producer_payload_reaches_worker_without_persisting_token(monkeypatch):
- source = "https://example.larkoffice.com/docx/doxcnToken"
+ ("source", "preflight_name"),
+ [
+ ("https://example.larkoffice.com/docx/doxcnToken", None),
+ ("https://example.larkoffice.com/sheets/shtcnToken", "Sheet Title"),
+ ("https://example.larkoffice.com/base/appToken?table=tblSales", "tblSales"),
+ ],
+)
+async def test_uat_producer_payload_reaches_worker_without_persisting_token(
+ monkeypatch,
+ source,
+ preflight_name,
+):
root_uri = "viking://resources/lark/doxcnToken"
submit_understanding = AsyncMock(return_value="response-1")
resource_processor = SimpleNamespace(
should_use_understanding_directly=lambda _source, **_kwargs: True,
submit_understanding=submit_understanding,
- tree_builder=SimpleNamespace(
- resolve_target_uri=AsyncMock(return_value=(root_uri, root_uri))
- ),
process_resource=AsyncMock(
return_value={
"status": "success",
@@ -457,6 +466,19 @@ async def test_uat_producer_payload_reaches_worker_without_persisting_token(monk
Mock(return_value=False),
)
monkeypatch.setattr("openviking.service.resource_service.uuid4", Mock(return_value="task-1"))
+
+ async def preflight(_self, _source, *, feishu_access_token=None):
+ assert feishu_access_token == "u-secret"
+ return SimpleNamespace(source_name=preflight_name, source_format="file")
+
+ async def plan_source_job_target(*, source_info, **_kwargs):
+ return root_uri, None, source_info.source_name is None
+
+ monkeypatch.setattr(
+ "openviking.parse.accessors.feishu_accessor.FeishuAccessor.preflight_source",
+ preflight,
+ )
+ service._plan_source_job_target = AsyncMock(side_effect=plan_source_job_target)
ctx = RequestContext(
user=UserIdentifier("account-1", "user-1"),
role=Role.USER,
@@ -471,8 +493,13 @@ async def test_uat_producer_payload_reaches_worker_without_persisting_token(monk
args={"feishu_access_token": "u-secret", "custom_option": "forwarded"},
)
- assert initial_result == {"status": "success", "task_id": "task-1"}
- assert task_tracker.create.await_args.kwargs["resource_id"] is None
+ expected_initial = {"status": "success", "task_id": "task-1"}
+ if preflight_name:
+ expected_initial["root_uri"] = root_uri
+ assert initial_result == expected_initial
+ assert task_tracker.create.await_args.kwargs["resource_id"] == (
+ None if preflight_name is None else root_uri
+ )
submit_understanding.assert_awaited_once_with(
source,
feishu_access_token="u-secret",
@@ -482,10 +509,7 @@ async def test_uat_producer_payload_reaches_worker_without_persisting_token(monk
assert queue_manager.enqueue.await_args.args[0] == QueueManager.EXTERNAL_PARSE
assert "u-secret" not in json.dumps(payload)
assert payload["understanding_response_id"] == "response-1"
- assert payload["args"] == {
- "custom_option": "forwarded",
- "parser_backend": "understanding",
- }
+ assert payload["args"] == {"custom_option": "forwarded"}
queued_msg = AddResourceMsg.from_dict(payload)
service._execute_resource_ingestion = AsyncMock(
@@ -508,31 +532,34 @@ async def test_uat_producer_payload_reaches_worker_without_persisting_token(monk
@pytest.mark.asyncio
-async def test_local_prepared_job_uses_add_resource_queue(monkeypatch):
+async def test_local_source_job_stages_snapshot_before_enqueue(monkeypatch, tmp_path):
root_uri = "viking://resources/script"
- resource_lock = {"lease_ref": "lock-1"}
- agfs = SimpleNamespace(
- pathlock_to_handoff=AsyncMock(return_value={"handle_id": "lock-1"}),
- pathlock_handoff=AsyncMock(),
- pathlock_release=AsyncMock(),
+ source_file = tmp_path / "script.md"
+ source_file.write_bytes(b"# durable source")
+ prepared_source = LocalResource(
+ path=source_file,
+ source_type=SourceType.LOCAL,
+ original_source=str(source_file),
+ is_temporary=False,
)
resource_processor = SimpleNamespace(
- process_resource=AsyncMock(
- return_value={
- "status": "success",
- "root_uri": root_uri,
- "_post_process": {"root_uri": root_uri},
- "_resource_lock": resource_lock,
- }
- )
+ prepare_durable_source=AsyncMock(return_value=prepared_source),
+ should_use_understanding_api=Mock(return_value=False),
+ process_resource=AsyncMock(),
+ )
+ viking_fs = SimpleNamespace(
+ create_temp_uri=Mock(return_value="viking://temp/account-1/source-task"),
+ write_file_bytes=AsyncMock(),
+ delete_temp=AsyncMock(),
)
service = ResourceService(
- viking_fs=SimpleNamespace(_async_agfs=agfs),
+ viking_fs=viking_fs,
resource_processor=resource_processor,
skill_processor=SimpleNamespace(),
)
service._enqueue_add_resource_job = AsyncMock(return_value=SimpleNamespace(task_id="task-1"))
+ service._plan_source_job_target = AsyncMock(return_value=(root_uri, None, False))
monkeypatch.setattr(
service,
"_connector_delegate",
@@ -548,7 +575,7 @@ async def test_local_prepared_job_uses_add_resource_queue(monkeypatch):
)
result = await service.add_resource(
- path="/tmp/script.md",
+ path=str(source_file),
ctx=ctx,
to=root_uri,
wait=False,
@@ -557,9 +584,44 @@ async def test_local_prepared_job_uses_add_resource_queue(monkeypatch):
assert result["task_id"] == "task-1"
call = service._enqueue_add_resource_job.await_args
assert call.kwargs["queue_name"] == QueueManager.ADD_RESOURCE
- assert call.args[0].prepared == {"root_uri": root_uri}
- assert call.args[0].lock_handoff == {"handle_id": "lock-1"}
- agfs.pathlock_to_handoff.assert_awaited_once_with(resource_lock)
+ message = call.args[0]
+ assert message.job_phase is AddResourcePhase.SOURCE
+ assert message.prepared is None
+ assert message.staged_source["original_source"] == str(source_file)
+ viking_fs.write_file_bytes.assert_awaited_once_with(
+ message.staged_source["source_uri"],
+ b"# durable source",
+ ctx=ctx,
+ )
+ resource_processor.process_resource.assert_not_awaited()
+
+ viking_fs.tree = AsyncMock(
+ return_value=[
+ {
+ "rel_path": "script.md",
+ "isDir": False,
+ "uri": message.staged_source["source_uri"],
+ }
+ ]
+ )
+ viking_fs.read_file_bytes = AsyncMock(return_value=b"# durable source")
+ service._execute_resource_ingestion = AsyncMock(
+ return_value={"status": "success", "root_uri": root_uri}
+ )
+
+ await service.execute_add_resource_job(
+ message,
+ ctx=ctx,
+ resource_lock=None,
+ stage_callback=AsyncMock(),
+ )
+
+ materialized = service._execute_resource_ingestion.await_args.kwargs["prepared_resource"]
+ try:
+ assert materialized.path.read_bytes() == b"# durable source"
+ assert materialized.original_source == str(source_file)
+ finally:
+ materialized.cleanup()
@pytest.mark.asyncio
@@ -630,6 +692,15 @@ async def test_uat_producer_cancellation_respects_queue_ownership(
"openviking.service.resource_service.is_git_repo_url",
Mock(return_value=False),
)
+
+ async def preflight(_self, _source, *, feishu_access_token=None):
+ assert feishu_access_token == "u-secret"
+ return SimpleNamespace(source_name=None, source_format="file")
+
+ monkeypatch.setattr(
+ "openviking.parse.accessors.feishu_accessor.FeishuAccessor.preflight_source",
+ preflight,
+ )
ctx = RequestContext(
user=UserIdentifier("account-1", "user-1"),
role=Role.USER,
@@ -645,14 +716,20 @@ async def test_uat_producer_cancellation_respects_queue_ownership(
args={"feishu_access_token": "u-secret"},
)
- release.assert_awaited_once_with(lock_lease)
- task_tracker.create.assert_not_awaited()
- task_tracker.fail.assert_not_awaited()
if cancel_stage == "submit":
+ release.assert_not_awaited()
+ task_tracker.create.assert_not_awaited()
+ task_tracker.fail.assert_not_awaited()
enqueue.assert_not_awaited()
else:
+ release.assert_awaited_once_with(lock_lease)
+ task_tracker.create.assert_awaited_once()
enqueue.assert_awaited_once()
assert enqueue.await_args.args[0] == QueueManager.EXTERNAL_PARSE
+ if cancel_stage == "enqueue":
+ task_tracker.fail.assert_awaited_once()
+ else:
+ task_tracker.fail.assert_not_awaited()
@pytest.mark.parametrize(
@@ -784,7 +861,9 @@ async def test_prepared_add_resource_job_forwards_processing_mode():
@pytest.mark.asyncio
-async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
+async def test_add_resource_processor_persists_final_uri_and_cleans_staged_source(
+ monkeypatch,
+):
final_uri = "viking://resources/真实文档标题"
service = SimpleNamespace(
execute_add_resource_job=AsyncMock(
@@ -798,6 +877,7 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
update_stage=AsyncMock(),
complete=AsyncMock(),
fail=AsyncMock(),
+ get_task_auth=AsyncMock(return_value={}),
wait_for_descendants=AsyncMock(),
)
monkeypatch.setattr(
@@ -809,11 +889,15 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
"openviking.storage.queuefs.get_queue_manager",
Mock(return_value=queue_manager),
)
+ viking_fs = SimpleNamespace(
+ _async_agfs=SimpleNamespace(pathlock_release=AsyncMock()),
+ delete_temp=AsyncMock(),
+ )
processor = AddResourceProcessor(
service,
asyncio.get_running_loop(),
QueueManager.ADD_RESOURCE,
- SimpleNamespace(_async_agfs=SimpleNamespace(pathlock_release=AsyncMock())),
+ viking_fs,
)
msg = AddResourceMsg(
task_id="task-1",
@@ -823,7 +907,13 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
user_id="user-1",
role="user",
defer_target_resolution=True,
- understanding_response_id="response-1",
+ staged_source={
+ "temp_uri": "viking://temp/account-1/task-1",
+ "source_uri": "viking://temp/account-1/task-1/source/document.md",
+ "source_type": "local",
+ "original_source": "/tmp/document.md",
+ "meta": {},
+ },
)
data = msg.to_dict()
@@ -876,6 +966,11 @@ async def test_add_resource_processor_persists_final_resource_uri(monkeypatch):
"user_id": "user-1",
"resource_id": final_uri,
}
+ viking_fs.delete_temp.assert_awaited_once()
+ cleanup_call = viking_fs.delete_temp.await_args
+ assert cleanup_call.args == ("viking://temp/account-1/task-1",)
+ assert cleanup_call.kwargs["ctx"].account_id == "account-1"
+ assert cleanup_call.kwargs["ctx"].user.user_id == "user-1"
assert await processor._requeue_lock_handoff(msg, RuntimeError("stale lock"))
assert queue_manager.enqueue.await_args.args[0] == QueueManager.ADD_RESOURCE
diff --git a/tests/service/test_resource_service_connector.py b/tests/service/test_resource_service_connector.py
index a8409e110..0f1be63f0 100644
--- a/tests/service/test_resource_service_connector.py
+++ b/tests/service/test_resource_service_connector.py
@@ -15,7 +15,7 @@ from openviking.parse.mode import ParseMode
from openviking.server.identity import RequestContext, Role
from openviking.service import resource_service as resource_service_module
from openviking.service.resource_service import ResourceService
-from openviking.storage.queuefs.add_resource_msg import AddResourceMsg
+from openviking.storage.queuefs.add_resource_msg import AddResourceMsg, AddResourcePhase
from openviking_cli.exceptions import InvalidArgumentError
from openviking_cli.session.user_id import UserIdentifier
@@ -715,7 +715,7 @@ async def test_git_credentials_with_include_never_enqueue_native_job(
connector_config.allowed_add_types = ["tos", "git"]
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
service._connector.submit = AsyncMock()
- service.enqueue_git_add_resource = AsyncMock()
+ service._enqueue_add_resource_job = AsyncMock()
with pytest.raises(InvalidArgumentError, match="cannot fall back") as exc_info:
await service.add_resource(
@@ -729,7 +729,7 @@ async def test_git_credentials_with_include_never_enqueue_native_job(
assert "ghp-secret" not in str(exc_info.value)
service._connector.submit.assert_not_awaited()
- service.enqueue_git_add_resource.assert_not_awaited()
+ service._enqueue_add_resource_job.assert_not_awaited()
@pytest.mark.asyncio
@@ -742,14 +742,17 @@ async def test_git_credentials_with_include_never_enqueue_native_job(
ids=["token", "username"],
)
async def test_native_git_enqueue_rejects_credentials_before_durable_job(
+ monkeypatch,
ctx,
service,
credential_args,
):
+ monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
+ service._connector.should_delegate = Mock(return_value=False)
service._enqueue_add_resource_job = AsyncMock()
- with pytest.raises(InvalidArgumentError, match="persists job parameters") as exc_info:
- await service.enqueue_git_add_resource(
+ with pytest.raises(InvalidArgumentError, match="must use args.auth_config") as exc_info:
+ await service.add_resource(
path="https://git.example/org/private.git",
ctx=ctx,
to="viking://resources/private",
@@ -761,21 +764,30 @@ async def test_native_git_enqueue_rejects_credentials_before_durable_job(
@pytest.mark.asyncio
-async def test_native_git_nested_auth_runs_request_local_before_durable_queue(
+async def test_native_git_nested_auth_is_owned_by_durable_task(
monkeypatch,
connector_config,
ctx,
service,
):
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
+ service._connector.should_delegate = Mock(return_value=False)
+ service._preflight_git_source = AsyncMock(
+ return_value=SimpleNamespace(
+ source_name="private",
+ source_path="https://git.example/org/private.git",
+ source_format="repository",
+ )
+ )
+ service._plan_source_job_target = AsyncMock(
+ return_value=("viking://resources/private", None, False)
+ )
service._execute_resource_ingestion = AsyncMock(
- return_value={
- "status": "success",
- "root_uri": "viking://resources/private",
- "task_id": "task-private",
- }
+ side_effect=AssertionError("Git source work must run in the queue worker")
+ )
+ service._enqueue_add_resource_job = AsyncMock(
+ return_value=SimpleNamespace(task_id="task-private")
)
- service.enqueue_git_add_resource = AsyncMock()
result = await service.add_resource(
path="https://git.example/org/private.git",
@@ -788,29 +800,40 @@ async def test_native_git_nested_auth_runs_request_local_before_durable_queue(
)
assert result["task_id"] == "task-private"
- service.enqueue_git_add_resource.assert_not_awaited()
- call = service._execute_resource_ingestion.await_args.kwargs
- assert call["defer_post_processing"] is True
- assert call["branch"] == "main"
- assert call["auth_config"] == {"username": "oauth2", "token": "secret-token"}
+ service._execute_resource_ingestion.assert_not_awaited()
+ message = service._enqueue_add_resource_job.await_args.args[0]
+ assert message.args == {"branch": "main"}
+ assert "secret-token" not in json.dumps(message.to_dict())
+ assert service._enqueue_add_resource_job.await_args.kwargs["task_auth"] == {
+ "provider": "git_http_basic",
+ "username": "oauth2",
+ "token": "secret-token",
+ "repo_url": "https://git.example/org/private.git",
+ }
@pytest.mark.asyncio
-async def test_native_git_nested_auth_watch_uses_private_auth_state(
+async def test_native_git_nested_auth_watch_is_copied_from_task_state_by_worker(
monkeypatch,
connector_config,
ctx,
service,
):
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
- service._execute_resource_ingestion = AsyncMock(
- return_value={
- "status": "success",
- "root_uri": "viking://resources/private",
- "task_id": "task-private",
- }
+ service._connector.should_delegate = Mock(return_value=False)
+ service._preflight_git_source = AsyncMock(
+ return_value=SimpleNamespace(
+ source_name="private",
+ source_path="https://git.example/org/private.git",
+ source_format="repository",
+ )
+ )
+ service._plan_source_job_target = AsyncMock(
+ return_value=("viking://resources/private", None, False)
+ )
+ service._enqueue_add_resource_job = AsyncMock(
+ return_value=SimpleNamespace(task_id="task-private")
)
- service.enqueue_git_add_resource = AsyncMock()
result = await service.add_resource(
path="https://git.example/org/private.git",
@@ -821,33 +844,39 @@ async def test_native_git_nested_auth_watch_uses_private_auth_state(
)
assert result["task_id"] == "task-private"
- call = service._execute_resource_ingestion.await_args.kwargs
- assert call["auth_config"] == {"username": "oauth2", "token": "secret-token"}
- assert call["watch_auth_state"] == {
+ message = service._enqueue_add_resource_job.await_args.args[0]
+ assert message.watch_interval == 5
+ assert "secret-token" not in json.dumps(message.to_dict())
+ assert service._enqueue_add_resource_job.await_args.kwargs["task_auth"] == {
"provider": "git_http_basic",
"username": "oauth2",
"token": "secret-token",
"repo_url": "https://git.example/org/private.git",
}
- service.enqueue_git_add_resource.assert_not_awaited()
@pytest.mark.asyncio
-async def test_native_git_watch_refresh_consumes_restored_auth_before_queue(
+async def test_native_git_watch_refresh_queues_with_restored_task_auth(
monkeypatch,
connector_config,
ctx,
service,
):
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
- service._execute_resource_ingestion = AsyncMock(
- return_value={
- "status": "success",
- "root_uri": "viking://resources/private",
- "task_id": "task-refresh",
- }
+ service._connector.should_delegate = Mock(return_value=False)
+ service._preflight_git_source = AsyncMock(
+ return_value=SimpleNamespace(
+ source_name="private",
+ source_path="https://git.example/org/private.git",
+ source_format="repository",
+ )
+ )
+ service._plan_source_job_target = AsyncMock(
+ return_value=("viking://resources/private", None, False)
+ )
+ service._enqueue_add_resource_job = AsyncMock(
+ return_value=SimpleNamespace(task_id="task-refresh")
)
- service.enqueue_git_add_resource = AsyncMock()
result = await service.refresh_resource(
path="https://git.example/org/private.git",
@@ -858,22 +887,37 @@ async def test_native_git_watch_refresh_consumes_restored_auth_before_queue(
)
assert result["task_id"] == "task-refresh"
- call = service._execute_resource_ingestion.await_args.kwargs
- assert call["manage_watch"] is False
- assert call["auth_config"] == {"username": "oauth2", "token": "secret-token"}
- service.enqueue_git_add_resource.assert_not_awaited()
+ message = service._enqueue_add_resource_job.await_args.args[0]
+ assert message.skip_watch_management is True
+ assert "secret-token" not in json.dumps(message.to_dict())
+ assert service._enqueue_add_resource_job.await_args.kwargs["task_auth"] == {
+ "provider": "git_http_basic",
+ "username": "oauth2",
+ "token": "secret-token",
+ "repo_url": "https://git.example/org/private.git",
+ }
@pytest.mark.asyncio
-async def test_native_git_enqueue_rejects_nested_auth_without_secret_in_error(ctx, service):
+async def test_native_git_rejects_malformed_nested_auth_without_secret_in_error(
+ monkeypatch, ctx, service
+):
+ monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
+ service._connector.should_delegate = Mock(return_value=False)
service._enqueue_add_resource_job = AsyncMock()
- with pytest.raises(InvalidArgumentError, match="auth_config") as exc_info:
- await service.enqueue_git_add_resource(
+ with pytest.raises(InvalidArgumentError, match="unsupported fields") as exc_info:
+ await service.add_resource(
path="https://git.example/org/private.git",
ctx=ctx,
to="viking://resources/private",
- args={"auth_config": {"username": "oauth2", "token": "secret-token"}},
+ args={
+ "auth_config": {
+ "username": "oauth2",
+ "token": "secret-token",
+ "password": "another-secret",
+ }
+ },
)
assert "secret-token" not in str(exc_info.value)
@@ -881,29 +925,24 @@ async def test_native_git_enqueue_rejects_nested_auth_without_secret_in_error(ct
@pytest.mark.asyncio
-async def test_git_url_embedded_credentials_are_passed_to_native_queue_unchanged(
+async def test_git_url_embedded_credentials_are_rejected_before_enqueue(
monkeypatch,
connector_config,
ctx,
service,
):
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
- service.enqueue_git_add_resource = AsyncMock(
- return_value={
- "status": "success",
- "root_uri": "viking://resources/private",
- "task_id": "task-private",
- }
- )
+ service._enqueue_add_resource_job = AsyncMock()
source = "https://user:[email protected]/org/private.git"
- result = await service.add_resource(
- path=source,
- ctx=ctx,
- to="viking://resources/private",
- )
+ with pytest.raises(InvalidArgumentError, match="cannot contain userinfo") as exc_info:
+ await service.add_resource(
+ path=source,
+ ctx=ctx,
+ to="viking://resources/private",
+ )
- assert result["task_id"] == "task-private"
- assert service.enqueue_git_add_resource.await_args.kwargs["path"] == source
+ assert "embedded-token" not in str(exc_info.value)
+ service._enqueue_add_resource_job.assert_not_awaited()
def test_prepared_add_resource_payload_drops_nested_git_auth():
@@ -921,11 +960,12 @@ def test_prepared_add_resource_payload_drops_nested_git_auth():
payload = msg.to_dict()
assert payload["args"] == {}
+ assert payload["job_phase"] == AddResourcePhase.POST_PROCESS
assert "secret-token" not in json.dumps(payload)
@pytest.mark.asyncio
-async def test_request_local_git_auth_is_absent_from_real_prepared_queue_message(ctx, service):
+async def test_postprocess_queue_message_drops_git_auth(ctx, service):
service._resource_processor = SimpleNamespace(
process_resource=AsyncMock(
return_value={
@@ -980,7 +1020,19 @@ async def test_add_resource_falls_back_for_shared_source_with_parent(
connector_config.allowed_add_types = ["tos", "git"]
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
service._connector.submit = AsyncMock()
- service.enqueue_git_add_resource = AsyncMock(return_value={"root_uri": "standard-pipeline"})
+ service._preflight_git_source = AsyncMock(
+ return_value=SimpleNamespace(
+ source_name="repo",
+ source_path="https://git.example/org/repo.git",
+ source_format="repository",
+ )
+ )
+ service._plan_source_job_target = AsyncMock(
+ return_value=("viking://resources/repo/repo", None, False)
+ )
+ service._enqueue_add_resource_job = AsyncMock(
+ return_value=SimpleNamespace(task_id="task-standard")
+ )
result = await service.add_resource(
path="https://git.example/org/repo.git",
@@ -988,9 +1040,13 @@ async def test_add_resource_falls_back_for_shared_source_with_parent(
parent="viking://resources/repo",
)
- assert result == {"root_uri": "standard-pipeline"}
+ assert result == {
+ "status": "success",
+ "task_id": "task-standard",
+ "root_uri": "viking://resources/repo/repo",
+ }
service._connector.submit.assert_not_awaited()
- service.enqueue_git_add_resource.assert_awaited_once()
+ service._enqueue_add_resource_job.assert_awaited_once()
@pytest.mark.asyncio
@@ -1023,7 +1079,6 @@ async def test_add_resource_job_executes_frozen_route(ctx, service):
service._execute_resource_ingestion = AsyncMock(
return_value={"status": "success", "root_uri": "root"}
)
- service._should_use_connector = Mock()
resource_lock = SimpleNamespace(close=AsyncMock())
stage_callback = AsyncMock()
msg = AddResourceMsg(
@@ -1050,7 +1105,6 @@ async def test_add_resource_job_executes_frozen_route(ctx, service):
assert call["defer_post_processing"] is False
assert call["resource_lock"] is resource_lock
assert call["parser_backend"] == "understanding"
- service._should_use_connector.assert_not_called()
@pytest.mark.asyncio
@@ -1063,7 +1117,7 @@ async def test_shared_source_create_parent_false_routes_to_connector(
connector_config.allowed_add_types = ["tos", "git"]
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
service._connector.submit = AsyncMock(return_value={"status": "accepted"})
- service.enqueue_git_add_resource = AsyncMock()
+ service._enqueue_add_resource_job = AsyncMock()
result = await service.add_resource(
path="https://git.example/org/repo.git",
@@ -1073,7 +1127,7 @@ async def test_shared_source_create_parent_false_routes_to_connector(
)
assert result == {"status": "accepted"}
- service.enqueue_git_add_resource.assert_not_awaited()
+ service._enqueue_add_resource_job.assert_not_awaited()
service._connector.submit.assert_awaited_once()
@@ -1224,7 +1278,7 @@ async def test_git_reason_routes_to_connector(
connector_config.allowed_add_types = ["tos", "git"]
monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
service._connector.submit = AsyncMock(return_value={"status": "accepted"})
- service.enqueue_git_add_resource = AsyncMock()
+ service._enqueue_add_resource_job = AsyncMock()
result = await service.add_resource(
path="https://git.example/org/repo.git",
@@ -1234,7 +1288,7 @@ async def test_git_reason_routes_to_connector(
)
assert result == {"status": "accepted"}
- service.enqueue_git_add_resource.assert_not_awaited()
+ service._enqueue_add_resource_job.assert_not_awaited()
connector_kwargs = service._connector.submit.await_args.kwargs
assert connector_kwargs["reason"] == "track quarterly reports"
diff --git a/tests/service/test_resource_service_parse_mode.py b/tests/service/test_resource_service_parse_mode.py
index ae0b6c2e9..fb045d9b2 100644
--- a/tests/service/test_resource_service_parse_mode.py
+++ b/tests/service/test_resource_service_parse_mode.py
@@ -31,6 +31,12 @@ class _ResourceProcessor:
"_post_process": {"root_is_file": self.root_is_file},
}
+ async def prepare_durable_source(self, *_args, **_kwargs):
+ return None
+
+ async def finish_prepared_resource(self, *_args, **_kwargs):
+ return {"status": "success"}
+
class _DiscardedBackgroundTask:
def add_done_callback(self, callback):
@@ -75,6 +81,11 @@ def service(monkeypatch: pytest.MonkeyPatch) -> ResourceService:
"_enqueue_add_resource_job",
AsyncMock(return_value=SimpleNamespace(task_id="task-1")),
)
+ monkeypatch.setattr(
+ instance,
+ "_plan_source_job_target",
+ AsyncMock(return_value=("viking://resources/test", None, False)),
+ )
return instance
@@ -115,13 +126,30 @@ async def test_no_split_watch_replay_preserves_auto_bound_file_target(
scheduler = WatchScheduler(resource_service=service, check_interval=1)
scheduler._watch_manager = watch_manager
service._watch_scheduler = scheduler
+ service._plan_source_job_target = AsyncMock(
+ side_effect=lambda **kwargs: (
+ "viking://resources/test",
+ None,
+ kwargs["defer_candidate_resolution"],
+ )
+ )
- await service.add_resource(
+ result = await service.add_resource(
path="https://example.com/guide.md",
ctx=ctx,
watch_interval=30.0,
args={"parse_mode": "no_split"},
)
+ assert "root_uri" not in result
+ message = service._enqueue_add_resource_job.await_args.args[0]
+ assert message.defer_target_resolution is True
+ await service.execute_add_resource_job(
+ message,
+ ctx=ctx,
+ resource_lock=None,
+ stage_callback=AsyncMock(),
+ task_auth={},
+ )
tasks = await watch_manager.get_all_tasks(
account_id=ctx.account_id,
@@ -136,9 +164,35 @@ async def test_no_split_watch_replay_preserves_auto_bound_file_target(
await scheduler._execute_task(task)
- processor = service._resource_processor
- assert processor.calls[-1]["to"] == "viking://resources/test"
- assert processor.calls[-1]["to_is_directory"] is False
+ replay_message = service._enqueue_add_resource_job.await_args.args[0]
+ assert replay_message.root_uri == "viking://resources/test"
+ assert replay_message.to_is_directory is False
+
+
+async def test_no_split_defers_initial_root_when_parent_targeted(
+ service: ResourceService,
+ ctx: RequestContext,
+):
+ service._plan_source_job_target = AsyncMock(
+ side_effect=lambda **kwargs: (
+ "viking://resources/test",
+ None,
+ kwargs["defer_candidate_resolution"],
+ )
+ )
+
+ result = await service.add_resource(
+ path="https://example.com/guide.md",
+ ctx=ctx,
+ parent="viking://resources/docs",
+ args={"parse_mode": "no_split"},
+ )
+
+ assert "root_uri" not in result
+ assert service._plan_source_job_target.await_args.kwargs["defer_candidate_resolution"] is True
+ message = service._enqueue_add_resource_job.await_args.args[0]
+ assert message.defer_target_resolution is True
@pytest.mark.asyncio
@@ -157,6 +211,14 @@ async def test_no_split_watch_persists_auto_bound_multi_artifact_directory(
watch_interval=30.0,
args={"parse_mode": "no_split"},
)
+ message = service._enqueue_add_resource_job.await_args.args[0]
+ await service.execute_add_resource_job(
+ message,
+ ctx=ctx,
+ resource_lock=None,
+ stage_callback=AsyncMock(),
+ task_auth={},
+ )
tasks = await watch_manager.get_all_tasks(
account_id=ctx.account_id,
@@ -266,7 +328,7 @@ async def test_no_split_bypasses_understanding_shortcut(
raising=False,
)
- await service.add_resource(
+ result = await service.add_resource(
path="https://example.com/manual.pdf",
ctx=ctx,
to="viking://resources/manual",
@@ -274,33 +336,39 @@ async def test_no_split_bypasses_understanding_shortcut(
allow_local_path_resolution=False,
)
+ assert result["root_uri"] == "viking://resources/test"
direct_probe.assert_not_called()
api_probe.assert_not_called()
@pytest.mark.asyncio
async def test_native_git_enqueue_persists_internal_no_split_mode(
+ monkeypatch: pytest.MonkeyPatch,
service: ResourceService,
ctx: RequestContext,
):
+ monkeypatch.setattr(resource_service_module, "is_git_repo_url", lambda _path: True)
service._preflight_git_source = AsyncMock(
- return_value=SimpleNamespace(source_name="repo", source_path=None)
- )
- service._plan_resource_target = AsyncMock(
- return_value=("viking://resources/repo", None)
+ return_value=SimpleNamespace(
+ source_name="repo",
+ source_path="https://github.com/volcengine/OpenViking.git",
+ source_format="repository",
+ )
)
- service._enqueue_add_resource_job = AsyncMock(
- return_value=SimpleNamespace(task_id="task-git")
+ service._plan_source_job_target = AsyncMock(
+ return_value=("viking://resources/repo", None, False)
)
+ service._enqueue_add_resource_job = AsyncMock(return_value=SimpleNamespace(task_id="task-git"))
- result = await service.enqueue_git_add_resource(
+ result = await service.add_resource(
path="https://github.com/volcengine/OpenViking.git",
ctx=ctx,
to="viking://resources/repo",
- parse_mode="no_split",
+ args={"parse_mode": "no_split"},
)
assert result["task_id"] == "task-git"
message = service._enqueue_add_resource_job.await_args.args[0]
assert isinstance(message, AddResourceMsg)
assert message.parse_mode == "no_split"
+ assert "parser_backend" not in message.args
diff --git a/tests/service/test_resource_service_understanding_routing.py b/tests/service/test_resource_service_understanding_routing.py
index b352b6f81..21243ffc0 100644
--- a/tests/service/test_resource_service_understanding_routing.py
+++ b/tests/service/test_resource_service_understanding_routing.py
@@ -51,9 +51,8 @@ async def test_extensionless_remote_url_queues_frozen_understanding_route(
pathlock_release=AsyncMock(),
)
processor = SimpleNamespace(
- understanding_api_enabled=lambda: True,
should_use_understanding_directly=lambda _source, **_kwargs: False,
- prepare_resource=AsyncMock(return_value=prepared),
+ prepare_durable_source=AsyncMock(return_value=prepared),
should_use_understanding_api=lambda resource: resource is prepared,
submit_understanding=AsyncMock(return_value="response-1"),
tree_builder=SimpleNamespace(
@@ -106,7 +105,7 @@ async def test_extensionless_remote_url_queues_frozen_understanding_route(
_, message = queue_manager.enqueue.await_args.args
assert queue_manager.enqueue.await_args.args[0] == QueueManager.EXTERNAL_PARSE
queued = AddResourceMsg.from_dict(message)
- assert queued.args["parser_backend"] == "understanding"
+ assert "parser_backend" not in queued.args
assert queued.args["resolved_extension"] == ".pdf"
assert queued.understanding_response_id == "response-1"
assert queued.source_name == "manual.pdf"
@@ -153,15 +152,12 @@ async def test_remote_mpeg_ts_url_queues_understanding_after_prepare(
)
)
processor = SimpleNamespace(
- understanding_api_enabled=lambda: True,
should_use_understanding_directly=lambda _source, **_kwargs: False,
- prepare_resource=AsyncMock(return_value=prepared),
+ prepare_durable_source=AsyncMock(return_value=prepared),
should_use_understanding_api=lambda resource: resource is prepared,
submit_understanding=AsyncMock(return_value="response-1"),
tree_builder=SimpleNamespace(resolve_target_uri=resolve_target_uri),
- reserve_unique_candidate=AsyncMock(
- return_value=("viking://resources/video/sample", lock)
- ),
+ reserve_unique_candidate=AsyncMock(return_value=("viking://resources/video/sample", lock)),
process_resource=AsyncMock(),
)
service = ResourceService(
@@ -170,7 +166,7 @@ async def test_remote_mpeg_ts_url_queues_understanding_after_prepare(
resource_processor=processor,
skill_processor=object(),
)
- service._should_use_connector = lambda *_args, **_kwargs: False
+ service._connector_delegate = SimpleNamespace(should_delegate=lambda *_args, **_kwargs: False)
tracker = SimpleNamespace(
create=AsyncMock(return_value=SimpleNamespace(task_id="task-1")),
update_stage=AsyncMock(),
@@ -199,7 +195,7 @@ async def test_remote_mpeg_ts_url_queues_understanding_after_prepare(
"root_uri": "viking://resources/video/sample",
"task_id": "task-1",
}
- processor.prepare_resource.assert_awaited_once()
+ processor.prepare_durable_source.assert_awaited_once()
processor.process_resource.assert_not_awaited()
processor.submit_understanding.assert_awaited_once_with(prepared)
resolve_target_uri.assert_awaited_once()
@@ -207,7 +203,7 @@ async def test_remote_mpeg_ts_url_queues_understanding_after_prepare(
_, message = queue_manager.enqueue.await_args.args
assert queue_manager.enqueue.await_args.args[0] == QueueManager.EXTERNAL_PARSE
queued = AddResourceMsg.from_dict(message)
- assert queued.args["parser_backend"] == "understanding"
+ assert "parser_backend" not in queued.args
assert queued.args["resolved_extension"] == "mpegts"
assert queued.understanding_response_id == "response-1"
assert queued.source_name == "sample.ts"
diff --git a/tests/service/test_resource_service_watch.py b/tests/service/test_resource_service_watch.py
index dd20ba04c..248eb5670 100644
--- a/tests/service/test_resource_service_watch.py
+++ b/tests/service/test_resource_service_watch.py
@@ -49,6 +49,12 @@ class MockResourceProcessor:
),
}
+ def should_use_understanding_directly(self, *_args, **_kwargs):
+ return False
+
+ async def finish_prepared_resource(self, *_args, **_kwargs):
+ return {"status": "success"}
+
class MockSkillProcessor:
"""Mock SkillProcessor for testing."""
@@ -455,6 +461,16 @@ class TestAddResourceArgs:
)
disable_task_tracker(monkeypatch)
to_uri = "viking://resources/feishu_user_watch"
+ resource_service._plan_source_job_target = AsyncMock(return_value=(to_uri, None, False))
+
+ async def preflight(_self, _source, *, feishu_access_token=None):
+ assert feishu_access_token == "u-test"
+ return SimpleNamespace(source_name=None, source_format="file")
+
+ monkeypatch.setattr(
+ "openviking.parse.accessors.feishu_accessor.FeishuAccessor.preflight_source",
+ preflight,
+ )
await resource_service.add_resource(
path="https://example.feishu.cn/docx/doc_token",
@@ -467,6 +483,24 @@ class TestAddResourceArgs:
},
)
+ enqueue_call = resource_service._enqueue_add_resource_job.await_args
+ message = enqueue_call.args[0]
+ task_auth = enqueue_call.kwargs["task_auth"]
+ assert "u-test" not in str(message.to_dict())
+ assert task_auth == {
+ "provider": "feishu",
+ "access_token": "u-test",
+ "refresh_token": "r-test",
+ "expires_at": None,
+ }
+ await resource_service.execute_add_resource_job(
+ message,
+ ctx=request_context,
+ resource_lock=None,
+ stage_callback=AsyncMock(),
+ task_auth=task_auth,
+ )
+
processor = resource_service._resource_processor
assert processor.calls[-1]["feishu_access_token"] == "u-test"
assert "feishu_refresh_token" not in processor.calls[-1]
@@ -493,6 +527,14 @@ class TestAddResourceArgs:
disable_task_tracker(monkeypatch)
repo_url = "https://git.example/org/private.git"
to_uri = "viking://resources/git_private_watch"
+ resource_service._preflight_git_source = AsyncMock(
+ return_value=SimpleNamespace(
+ source_name="private",
+ source_path=repo_url,
+ source_format="repository",
+ )
+ )
+ resource_service._plan_source_job_target = AsyncMock(return_value=(to_uri, None, False))
await resource_service.add_resource(
path=repo_url,
@@ -508,6 +550,24 @@ class TestAddResourceArgs:
},
)
+ enqueue_call = resource_service._enqueue_add_resource_job.await_args
+ message = enqueue_call.args[0]
+ task_auth = enqueue_call.kwargs["task_auth"]
+ assert "git-secret" not in str(message.to_dict())
+ assert task_auth == {
+ "provider": "git_http_basic",
+ "username": "git-user",
+ "token": "git-secret",
+ "repo_url": repo_url,
+ }
+ await resource_service.execute_add_resource_job(
+ message,
+ ctx=request_context,
+ resource_lock=None,
+ stage_callback=AsyncMock(),
+ task_auth=task_auth,
+ )
+
processor = resource_service._resource_processor
assert processor.calls[-1]["auth_config"] == {
"username": "git-user",
@@ -516,7 +576,10 @@ class TestAddResourceArgs:
task = await get_task_by_uri(resource_service, to_uri, request_context)
assert task is not None
- assert task.processor_kwargs == {"branch": "main"}
+ assert task.processor_kwargs == {
+ "branch": "main",
+ "source_name": "private",
+ }
assert task.auth_state == {
"provider": "git_http_basic",
"username": "git-user",
diff --git a/tests/service/test_watch_recovery.py b/tests/service/test_watch_recovery.py
index 354370eae..c183e72b0 100644
--- a/tests/service/test_watch_recovery.py
+++ b/tests/service/test_watch_recovery.py
@@ -317,7 +317,6 @@ class TestResourceExistenceCheck:
skill_processor=MockSkillProcessor(),
watch_scheduler=None,
)
-
scheduler = WatchScheduler(
resource_service=resource_service,
viking_fs=None,
@@ -358,6 +357,9 @@ class TestResourceExistenceCheck:
skill_processor=MockSkillProcessor(),
watch_scheduler=None,
)
+ resource_service.refresh_resource = AsyncMock(
+ return_value={"root_uri": "viking://resources/existing"}
+ )
scheduler = WatchScheduler(
resource_service=resource_service,
@@ -395,6 +397,9 @@ class TestResourceExistenceCheck:
skill_processor=MockSkillProcessor(),
watch_scheduler=None,
)
+ resource_service.refresh_resource = AsyncMock(
+ return_value={"root_uri": "viking://resources/url"}
+ )
scheduler = WatchScheduler(
resource_service=resource_service,
@@ -416,7 +421,7 @@ class TestResourceExistenceCheck:
updated_task = await watch_manager.get_task(task.task_id)
assert updated_task is not None
assert updated_task.is_active is True
- assert resource_processor.call_count == 1
+ resource_service.refresh_resource.assert_awaited_once()
@pytest.mark.asyncio
async def test_feishu_user_token_watch_refreshes_before_execution(
@@ -430,6 +435,9 @@ class TestResourceExistenceCheck:
skill_processor=MockSkillProcessor(),
watch_scheduler=None,
)
+ resource_service.refresh_resource = AsyncMock(
+ return_value={"root_uri": "viking://resources/feishu-user-watch"}
+ )
scheduler = WatchScheduler(resource_service=resource_service, viking_fs=None)
await scheduler.start()
scheduler._feishu_oauth_client = FakeFeishuOAuthClient()
@@ -450,8 +458,7 @@ class TestResourceExistenceCheck:
await scheduler._execute_task(task)
assert scheduler._feishu_oauth_client.calls == ["r-old"]
- assert resource_processor.call_count == 1
- assert resource_processor.calls[-1]["feishu_access_token"] == "u-new"
+ assert resource_service.refresh_resource.await_args.kwargs["feishu_access_token"] == "u-new"
updated_task = await watch_manager.get_task(task.task_id)
assert updated_task is not None
@@ -460,7 +467,7 @@ class TestResourceExistenceCheck:
assert updated_task.auth_state["expires_at"] is not None
@pytest.mark.asyncio
- async def test_git_token_watch_restores_request_local_auth_for_execution(
+ async def test_git_token_watch_restores_task_auth_for_refresh(
self, temp_storage: Path, request_context: RequestContext
):
resource_service = ResourceService(
@@ -519,6 +526,9 @@ class TestSchedulerIntegration:
skill_processor=MockSkillProcessor(),
watch_scheduler=None,
)
+ resource_service.refresh_resource = AsyncMock(
+ return_value={"root_uri": "viking://resources/test"}
+ )
scheduler = WatchScheduler(
resource_service=resource_service,
@@ -540,7 +550,7 @@ class TestSchedulerIntegration:
await scheduler.stop()
- assert resource_processor.call_count >= 1
+ assert resource_service.refresh_resource.await_count >= 1
@pytest.mark.asyncio
async def test_scheduler_handles_multiple_tasks_after_restart(
@@ -561,6 +571,9 @@ class TestSchedulerIntegration:
skill_processor=MockSkillProcessor(),
watch_scheduler=None,
)
+ resource_service.refresh_resource = AsyncMock(
+ return_value={"root_uri": "viking://resources/test"}
+ )
scheduler = WatchScheduler(
resource_service=resource_service,
@@ -588,7 +601,7 @@ class TestSchedulerIntegration:
await scheduler.stop()
- assert resource_processor.call_count >= 2
+ assert resource_service.refresh_resource.await_count >= 2
@pytest.mark.asyncio
async def test_scheduler_skips_inactive_tasks_after_restart(
diff --git a/tests/storage/test_viking_fs_tree.py b/tests/storage/test_viking_fs_tree.py
index 9033f3074..36b940a88 100644
--- a/tests/storage/test_viking_fs_tree.py
+++ b/tests/storage/test_viking_fs_tree.py
@@ -61,7 +61,8 @@ def make_entry(
):
"""Build a Rust-shaped TreeEntry dict for tests.
- ``rel_path`` is derived from ``path`` by stripping the account prefix, and
+ ``rel_path`` is normalized by ``patch_tree_env`` for tree traversal tests,
+ matching the binding contract where it is relative to the query root.
``name`` defaults to the last path component when not given.
"""
if name is None:
@@ -80,6 +81,25 @@ def make_entry(
}
+def _query_relative_path(path: str, query_root: str) -> str:
+ base = query_root.rstrip("/")
+ normalized = path.rstrip("/")
+ if normalized == base:
+ return ""
+ if normalized.startswith(f"{base}/"):
+ return normalized[len(base) + 1 :]
+ return path.replace("/local/test_account/", "")
+
+
+def _with_query_relative_paths(entries, query_root: str):
+ normalized_entries = []
+ for entry in entries:
+ normalized = dict(entry)
+ normalized["rel_path"] = _query_relative_path(str(entry["path"]), query_root)
+ normalized_entries.append(normalized)
+ return normalized_entries
+
+
def patch_visibility(monkeypatch, fs, *, is_accessible=True):
"""Patch ACL + URI mapping used by visibility checks."""
monkeypatch.setattr(
@@ -109,8 +129,8 @@ def patch_tree_env(
fake_tree = entries_or_fn
else:
- async def fake_tree(_path, **_kwargs):
- return entries_or_fn
+ async def fake_tree(path, **_kwargs):
+ return _with_query_relative_paths(entries_or_fn, path)
monkeypatch.setattr(fs._async_agfs, "tree_directory", fake_tree)
monkeypatch.setattr(fs, "_uri_to_path", lambda _uri, **_kwargs: uri_to_path)
@@ -415,7 +435,7 @@ async def test_tree_original_structure(monkeypatch, fs):
assert e["mode"] == 0o644
assert e["modTime"] == "2026-01-01T00:00:00Z"
assert e["isDir"] is False
- assert e["rel_path"] == "resources/a"
+ assert e["rel_path"] == "a"
assert e["uri"] == "viking://resources/a"
@@ -456,7 +476,9 @@ async def test_tree_original_dfs_order(monkeypatch, fs):
result = await fs._tree_original("viking://resources", ctx=_default_ctx())
assert result[0]["name"] == "sub"
assert result[0]["isDir"] is True
+ assert result[0]["rel_path"] == "sub"
assert result[1]["name"] == "file.txt"
+ assert result[1]["rel_path"] == "sub/file.txt"
# ── _tree_agent tests ──
diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.py
index a6518c73b..aa880568e 100644
--- a/tests/test_task_tracker.py
+++ b/tests/test_task_tracker.py
@@ -353,6 +353,7 @@ async def test_to_dict(tracker: TaskTracker):
task = await tracker.create(
"session_commit",
resource_id="s1",
+ auth={"provider": "git_http_basic", "password": "secret"},
**_owner_kwargs(),
)
d = task.to_dict()
@@ -368,6 +369,13 @@ async def test_to_dict(tracker: TaskTracker):
assert isinstance(d["updated_at_iso"], str)
assert "account_id" not in d
assert "user_id" not in d
+ assert "auth" not in d
+ assert await tracker.get_task_auth(task.task_id, **_owner_kwargs()) == {
+ "provider": "git_http_basic",
+ "password": "secret",
+ }
+ assert (await tracker.get(task.task_id, **_owner_kwargs())).auth == {}
+ assert (await tracker.list_tasks(**_owner_kwargs()))[0].auth == {}
# ── Sanitization ──
@@ -501,6 +509,7 @@ async def test_persistent_store_writes_task_record_json():
task = await tracker.create(
"add_resource",
resource_id="viking://resources/demo",
+ auth={"provider": "feishu", "access_token": "secret-token"},
**_owner_kwargs(),
)
@@ -512,8 +521,18 @@ async def test_persistent_store_writes_task_record_json():
assert payload["account_id"] == "acme"
assert payload["user_id"] == "alice"
assert payload["stage"] is None
+ assert payload["auth"] == {
+ "provider": "feishu",
+ "access_token": "secret-token",
+ }
assert "schema_version" not in payload
+ await tracker.complete(task.task_id, {"ok": True}, **_owner_kwargs())
+ terminal_payload = json.loads(
+ agfs.files[f"/local/acme/_system/tasks/alice/{task.task_id}.json"].decode("utf-8")
+ )
+ assert terminal_payload["auth"] == {}
+
async def test_persistent_store_keeps_tasktracker_tasks_dict():
tracker = TaskTracker(store=PersistentTaskStore(_FakeAgfs()))
@@ -524,7 +543,12 @@ async def test_persistent_store_keeps_tasktracker_tasks_dict():
async def test_persistent_store_survives_tracker_reset():
agfs = _FakeAgfs()
tracker1 = TaskTracker(store=PersistentTaskStore(agfs))
- task = await tracker1.create("session_commit", resource_id="sess-123", **_owner_kwargs())
+ task = await tracker1.create(
+ "session_commit",
+ resource_id="sess-123",
+ auth={"provider": "feishu", "access_token": "secret-token"},
+ **_owner_kwargs(),
+ )
await tracker1.start(task.task_id, account_id="acme", user_id="alice")
tracker2 = TaskTracker(store=PersistentTaskStore(agfs))
@@ -532,6 +556,11 @@ async def test_persistent_store_survives_tracker_reset():
assert loaded is not None
assert loaded.status == TaskStatus.RUNNING
+ assert loaded.auth == {}
+ assert await tracker2.get_task_auth(task.task_id, **_owner_kwargs()) == {
+ "provider": "feishu",
+ "access_token": "secret-token",
+ }
async def test_persistent_store_ignores_existing_task_dirs():