Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(api): validate vector-space watermark with Pydantic (#44258)
Yann Roberto91a94a9f6ee3
diff --git a/api/services/vector_space_admission_service.py b/api/services/vector_space_admission_service.pyindex de68bcf59dc..25e41262384 100644--- a/api/services/vector_space_admission_service.py+++ b/api/services/vector_space_admission_service.py@@ -6,6 +6,7 @@ from collections.abc import Mappingfrom dataclasses import dataclassfrom typing import Any+from pydantic import BaseModel, ConfigDict, Fieldfrom sqlalchemy.orm import Sessionfrom configs import dify_config@@ -37,6 +38,13 @@ _ERROR_PATTERN = re.compile(VECTOR_SPACE_ADMISSION_ERROR_CODE = "vector_space_estimate_exceeded"+class _VectorSpaceEstimateWatermark(BaseModel):+ model_config = ConfigDict(strict=True)++ projected_usage_bytes: int = Field(ge=0)+ document_ids: list[str]++class VectorSpaceAdmissionError(ValueError):def __init__(self, message: str):self.description = message@@ -392,17 +400,9 @@ class VectorSpaceAdmissionService:stored_usage_bytes = 0document_ids: set[str] = set()if raw_state:- state = json.loads(raw_state)- stored_usage_bytes = state.get("projected_usage_bytes")- raw_document_ids = state.get("document_ids")- if (- type(stored_usage_bytes) is not int- or stored_usage_bytes < 0- or not isinstance(raw_document_ids, list)- or not all(isinstance(item, str) for item in raw_document_ids)- ):- raise ValueError("Invalid vector-space estimate watermark")- document_ids = set(raw_document_ids)+ state = _VectorSpaceEstimateWatermark.model_validate_json(raw_state)+ stored_usage_bytes = state.projected_usage_bytes+ document_ids = set(state.document_ids)base_usage_bytes = max(current_usage_bytes, stored_usage_bytes)projected_usage_bytes = base_usage_bytesdiff --git a/api/tests/unit_tests/services/test_vector_space_admission_service.py b/api/tests/unit_tests/services/test_vector_space_admission_service.pyindex 903db9dff84..aba63034031 100644--- a/api/tests/unit_tests/services/test_vector_space_admission_service.py+++ b/api/tests/unit_tests/services/test_vector_space_admission_service.py@@ -434,6 +434,85 @@ def test_independent_services_use_watermark_without_double_counting_fresh_usage(assert redis.ttls["tenant:tenant-1:vector_space_estimate_watermark"] == 1800+ "raw_state",+ [+ pytest.param("{", id="malformed-json"),+ pytest.param("null", id="null-state"),+ pytest.param("[]", id="array-state"),+ pytest.param("1", id="scalar-state"),+ pytest.param("{}", id="missing-fields"),+ pytest.param('{"document_ids": []}', id="missing-usage"),+ pytest.param('{"projected_usage_bytes": 0}', id="missing-document-ids"),+ pytest.param('{"projected_usage_bytes": -1, "document_ids": []}', id="negative-usage"),+ pytest.param('{"projected_usage_bytes": true, "document_ids": []}', id="boolean-usage"),+ pytest.param('{"projected_usage_bytes": "1", "document_ids": []}', id="string-usage"),+ pytest.param('{"projected_usage_bytes": 1.0, "document_ids": []}', id="float-usage"),+ pytest.param('{"projected_usage_bytes": null, "document_ids": []}', id="null-usage"),+ pytest.param('{"projected_usage_bytes": 0, "document_ids": null}', id="null-document-ids"),+ pytest.param('{"projected_usage_bytes": 0, "document_ids": "document-1"}', id="string-document-ids"),+ pytest.param('{"projected_usage_bytes": 0, "document_ids": {}}', id="object-document-ids"),+ pytest.param('{"projected_usage_bytes": 0, "document_ids": ["document-1", 1]}', id="numeric-document-id"),+ pytest.param('{"projected_usage_bytes": 0, "document_ids": [null]}', id="null-document-id"),+ ],+)+def test_invalid_watermark_rejects_admission_without_overwriting_cache(sqlite_session: Session, raw_state: str) -> None:+ redis = _FakeRedis()+ watermark_key = "tenant:tenant-1:vector_space_estimate_watermark"+ redis.setex(watermark_key, 123, raw_state)++ with pytest.raises(VectorSpaceAdmissionError) as exc_info:+ _check_estimate(sqlite_session, CloudPlan.SANDBOX, 10, redis=redis)++ message = "Unable to reserve estimated vector storage right now. Please try again later."+ assert str(exc_info.value) == message+ assert exc_info.value.description == message+ assert redis.values == {watermark_key: raw_state}+ assert redis.ttls == {watermark_key: 123}+++ ("watermark", "expected_mb", "expected_document_ids"),+ [+ pytest.param(+ {"projected_usage_bytes": 0, "document_ids": []},+ 10,+ ["文档-1"],+ id="zero-usage",+ ),+ pytest.param(+ {"projected_usage_bytes": 40 * _MEBIBYTE, "document_ids": ["document-0"], "extra": "ignored"},+ 50,+ ["document-0", "文档-1"],+ id="extra-fields",+ ),+ pytest.param(+ {"projected_usage_bytes": 50 * _MEBIBYTE, "document_ids": ["文档-1", "文档-1"]},+ 50,+ ["文档-1"],+ id="duplicate-unicode-document-ids",+ ),+ ],+)+def test_valid_watermark_preserves_usage_and_document_ids(+ sqlite_session: Session,+ watermark: dict[str, object],+ expected_mb: int,+ expected_document_ids: list[str],+) -> None:+ redis = _FakeRedis()+ watermark_key = "tenant:tenant-1:vector_space_estimate_watermark"+ redis.setex(watermark_key, 123, json.dumps(watermark, ensure_ascii=False))++ _check_estimate(sqlite_session, CloudPlan.SANDBOX, 10, document_id="文档-1", redis=redis)++ assert json.loads(redis.values[watermark_key]) == {+ "projected_usage_bytes": expected_mb * _MEBIBYTE,+ "document_ids": expected_document_ids,+ }+ assert redis.ttls[watermark_key] == 1800++def test_fresh_usage_above_watermark_becomes_next_projection_base(sqlite_session: Session) -> None:redis = _FakeRedis()_check_estimate(