Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from openviking.parse.accessors.base import LocalResource, SourceType
from openviking.parse.parser_router import ParserRouter
from openviking.parse.parsers.media.utils import MPEG_TS_PACKET_SIZE, MPEG_TS_PROBE_BYTES
from openviking.parse.registry import ParserRegistry
from openviking.parse.understanding_api import PREPARED_FILE_ID_ARG, PREPARED_RESPONSE_ID_ARG
from openviking.utils.media_processor import UnifiedResourceProcessor
def mpeg_ts_probe() -> bytes:
content = bytearray(MPEG_TS_PROBE_BYTES)
for offset in range(0, MPEG_TS_PROBE_BYTES, MPEG_TS_PACKET_SIZE):
content[offset] = 0x47
return bytes(content)
def test_should_use_understanding_api_for_signed_video_url(monkeypatch):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=False,
extensions=["mp4"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
router = ParserRouter(parser_registry=object())
assert router.should_use_understanding_api(
"https://example.com/media/video.mp4?X-Tos-Signature=abc&X-Tos-Expires=60"
)
def test_resolved_extension_routes_extensionless_download(monkeypatch, tmp_path):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=False,
extensions=["pdf"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
downloaded = tmp_path / "download"
downloaded.write_bytes(b"%PDF-1.7")
resource = LocalResource(
path=downloaded,
source_type=SourceType.HTTP,
original_source="https://example.com/download?id=123",
meta={"extension": ".pdf"},
is_temporary=False,
)
processor = UnifiedResourceProcessor()
processor._set_resolved_identity(resource, source_name=None)
assert resource.meta["resolved_extension"] == ".pdf"
assert processor.should_use_understanding_api(resource)
def test_should_use_understanding_api_for_local_mpeg_ts(monkeypatch, tmp_path):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=False,
extensions=["mpegts"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
path = tmp_path / "video.ts"
path.write_bytes(mpeg_ts_probe())
resource = LocalResource(
path=path,
source_type=SourceType.HTTP,
original_source="https://example.com/video.ts",
meta={"extension": ".ts"},
is_temporary=False,
)
processor = UnifiedResourceProcessor()
processor._set_resolved_identity(resource, source_name=None)
assert resource.meta["resolved_extension"] == "mpegts"
assert processor.should_use_understanding_api(resource)
def test_should_use_understanding_api_for_typescript_when_ts_configured(monkeypatch, tmp_path):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=False,
extensions=["ts"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
path = tmp_path / "source.ts"
path.write_text("export const answer: number = 42;\n")
router = ParserRouter(parser_registry=object())
assert router.should_use_understanding_api(path)
def test_should_not_use_understanding_api_for_typescript_source(monkeypatch, tmp_path):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=False,
extensions=["mpegts"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
path = tmp_path / "source.ts"
path.write_text("export const answer: number = 42;\n")
router = ParserRouter(parser_registry=object())
assert not router.should_use_understanding_api(path)
def test_should_not_use_understanding_api_for_mpeg_ts_when_extension_disabled(
monkeypatch,
tmp_path,
):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=False,
extensions=["mp4"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
path = tmp_path / "video.ts"
path.write_bytes(mpeg_ts_probe())
resource = LocalResource(
path=path,
source_type=SourceType.HTTP,
original_source="https://example.com/video.ts",
meta={"extension": ".ts"},
is_temporary=False,
)
processor = UnifiedResourceProcessor()
processor._set_resolved_identity(resource, source_name=None)
assert resource.meta["resolved_extension"] == "mpegts"
assert not processor.should_use_understanding_api(resource)
def test_parser_registry_keeps_typescript_ts_on_text_fallback(tmp_path):
path = tmp_path / "source.ts"
path.write_text("export const answer: number = 42;\n")
assert ParserRegistry().get_parser_for_file(path) is None
def test_should_use_understanding_api_for_feishu_url(monkeypatch):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=True,
extensions=[],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
router = ParserRouter(parser_registry=object())
router._understanding_api = SimpleNamespace(
can_submit_url_directly=lambda _source, **kwargs: bool(kwargs.get("feishu_access_token"))
)
assert router.should_use_understanding_api("https://example.larkoffice.com/wiki/wikicnToken")
assert router.should_use_understanding_directly(
"https://example.larkoffice.com/wiki/wikicnToken",
feishu_access_token="u-test",
)
assert not router.should_use_understanding_api(
"https://larkoffice.com.evil.example/wiki/wikicnToken"
)
def test_feishu_url_flag_is_required_even_when_extension_matches(monkeypatch):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=False,
extensions=["pdf"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
router = ParserRouter(parser_registry=object())
assert not router.should_use_understanding_api(
"https://example.larkoffice.com/docx/doxcnToken.pdf"
)
def test_directories_never_route_to_understanding(monkeypatch, tmp_path):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=False,
extensions=["pdf"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
resource = LocalResource(
path=tmp_path,
source_type=SourceType.HTTP,
original_source="https://example.com/site",
meta={"resolved_extension": ".pdf"},
is_temporary=False,
)
assert not UnifiedResourceProcessor().should_use_understanding_api(resource)
@pytest.mark.asyncio
async def test_normalized_feishu_markdown_stays_internal(monkeypatch, tmp_path):
config = SimpleNamespace(
parser_api=SimpleNamespace(
enable=True,
enable_feishu_url=True,
extensions=["md"],
),
)
monkeypatch.setattr(
"openviking_cli.utils.config.open_viking_config.get_openviking_config",
lambda: config,
)
resource = LocalResource(
path=tmp_path / "document.md",
source_type=SourceType.FEISHU,
original_source="https://example.feishu.cn/docx/doc-token",
meta={"resolved_extension": ".md"},
is_temporary=False,
)
registry = SimpleNamespace(parse=AsyncMock(return_value=object()))
understanding = SimpleNamespace(parse=AsyncMock(return_value=object()))
router = ParserRouter(registry)
router._understanding_api = understanding
processor = UnifiedResourceProcessor()
processor._parser_router = router
assert not processor.should_use_understanding_api(resource)
await router.parse(
resource,
resolved_extension=".md",
parser_backend="understanding",
)
registry.parse.assert_awaited_once()
understanding.parse.assert_not_awaited()
@pytest.mark.asyncio
async def test_forced_resolved_extension_survives_worker_redownload(tmp_path):
downloaded = tmp_path / "redetected.docx"
downloaded.write_bytes(b"content")
resource = LocalResource(
path=downloaded,
source_type=SourceType.HTTP,
original_source="https://example.com/download",
meta={"resolved_extension": ".docx"},
is_temporary=False,
)
parser_router = SimpleNamespace(parse=AsyncMock(return_value=object()))
processor = UnifiedResourceProcessor(vlm_processor=object())
processor._parser_router = parser_router
await processor.process(
"https://example.com/download",
prepared_resource=resource,
resolved_extension=".pdf",
)
assert parser_router.parse.await_args.kwargs["resolved_extension"] == ".pdf"
@pytest.mark.asyncio
async def test_prepared_understanding_response_bypasses_remote_redownload():
result = object()
parser_router = SimpleNamespace(parse=AsyncMock(return_value=result))
accessor = SimpleNamespace(
access=AsyncMock(side_effect=AssertionError("prepared response must not redownload source"))
)
processor = UnifiedResourceProcessor(vlm_processor=object())
processor._parser_router = parser_router
processor._accessor_registry = accessor
actual = await processor.process(
"https://example.com/expiring-download",
resolved_extension=".pdf",
**{PREPARED_RESPONSE_ID_ARG: "response-1"},
)
assert actual is result
accessor.access.assert_not_awaited()
parser_router.parse.assert_awaited_once()
assert parser_router.parse.await_args.args == ("https://example.com/expiring-download",)
assert parser_router.parse.await_args.kwargs["parser_backend"] == "understanding"
assert parser_router.parse.await_args.kwargs[PREPARED_RESPONSE_ID_ARG] == "response-1"
@pytest.mark.asyncio
async def test_prepared_understanding_file_bypasses_local_reopen():
result = object()
parser_router = SimpleNamespace(parse=AsyncMock(return_value=result))
accessor = SimpleNamespace(
access=AsyncMock(side_effect=AssertionError("prepared file must not reopen source"))
)
processor = UnifiedResourceProcessor(vlm_processor=object())
processor._parser_router = parser_router
processor._accessor_registry = accessor
actual = await processor.process(
"/tmp/upload_already_cleaned.pdf",
source_name="uploaded.pdf",
resolved_extension=".pdf",
**{PREPARED_FILE_ID_ARG: "file-1"},
)
assert actual is result
accessor.access.assert_not_awaited()
parser_router.parse.assert_awaited_once()
assert parser_router.parse.await_args.args == ("/tmp/upload_already_cleaned.pdf",)
assert parser_router.parse.await_args.kwargs["parser_backend"] == "understanding"
assert parser_router.parse.await_args.kwargs[PREPARED_FILE_ID_ARG] == "file-1"