Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(api): validate vector-space watermark with Pydantic (#44258)

Yann Roberto91a94a9f6ee3
diff --git a/api/services/vector_space_admission_service.py b/api/services/vector_space_admission_service.py
index de68bcf59dc..25e41262384 100644
--- a/api/services/vector_space_admission_service.py
+++ b/api/services/vector_space_admission_service.py
@@ -6,6 +6,7 @@ from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any
+from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy.orm import Session
from configs import dify_config
@@ -37,6 +38,13 @@ _ERROR_PATTERN = re.compile(
VECTOR_SPACE_ADMISSION_ERROR_CODE = "vector_space_estimate_exceeded"
+class _VectorSpaceEstimateWatermark(BaseModel):
+ model_config = ConfigDict(strict=True)
+
+ projected_usage_bytes: int = Field(ge=0)
+ document_ids: list[str]
+
+
class VectorSpaceAdmissionError(ValueError):
def __init__(self, message: str):
self.description = message
@@ -392,17 +400,9 @@ class VectorSpaceAdmissionService:
stored_usage_bytes = 0
document_ids: set[str] = set()
if raw_state:
- state = json.loads(raw_state)
- stored_usage_bytes = state.get("projected_usage_bytes")
- raw_document_ids = state.get("document_ids")
- if (
- type(stored_usage_bytes) is not int
- or stored_usage_bytes < 0
- or not isinstance(raw_document_ids, list)
- or not all(isinstance(item, str) for item in raw_document_ids)
- ):
- raise ValueError("Invalid vector-space estimate watermark")
- document_ids = set(raw_document_ids)
+ state = _VectorSpaceEstimateWatermark.model_validate_json(raw_state)
+ stored_usage_bytes = state.projected_usage_bytes
+ document_ids = set(state.document_ids)
base_usage_bytes = max(current_usage_bytes, stored_usage_bytes)
projected_usage_bytes = base_usage_bytes
diff --git a/api/tests/unit_tests/services/test_vector_space_admission_service.py b/api/tests/unit_tests/services/test_vector_space_admission_service.py
index 903db9dff84..aba63034031 100644
--- a/api/tests/unit_tests/services/test_vector_space_admission_service.py
+++ b/api/tests/unit_tests/services/test_vector_space_admission_service.py
@@ -434,6 +434,85 @@ def test_independent_services_use_watermark_without_double_counting_fresh_usage(
assert redis.ttls["tenant:tenant-1:vector_space_estimate_watermark"] == 1800
+ "raw_state",
+ [
+ pytest.param("{", id="malformed-json"),
+ pytest.param("null", id="null-state"),
+ pytest.param("[]", id="array-state"),
+ pytest.param("1", id="scalar-state"),
+ pytest.param("{}", id="missing-fields"),
+ pytest.param('{"document_ids": []}', id="missing-usage"),
+ pytest.param('{"projected_usage_bytes": 0}', id="missing-document-ids"),
+ pytest.param('{"projected_usage_bytes": -1, "document_ids": []}', id="negative-usage"),
+ pytest.param('{"projected_usage_bytes": true, "document_ids": []}', id="boolean-usage"),
+ pytest.param('{"projected_usage_bytes": "1", "document_ids": []}', id="string-usage"),
+ pytest.param('{"projected_usage_bytes": 1.0, "document_ids": []}', id="float-usage"),
+ pytest.param('{"projected_usage_bytes": null, "document_ids": []}', id="null-usage"),
+ pytest.param('{"projected_usage_bytes": 0, "document_ids": null}', id="null-document-ids"),
+ pytest.param('{"projected_usage_bytes": 0, "document_ids": "document-1"}', id="string-document-ids"),
+ pytest.param('{"projected_usage_bytes": 0, "document_ids": {}}', id="object-document-ids"),
+ pytest.param('{"projected_usage_bytes": 0, "document_ids": ["document-1", 1]}', id="numeric-document-id"),
+ pytest.param('{"projected_usage_bytes": 0, "document_ids": [null]}', id="null-document-id"),
+ ],
+)
+def test_invalid_watermark_rejects_admission_without_overwriting_cache(sqlite_session: Session, raw_state: str) -> None:
+ redis = _FakeRedis()
+ watermark_key = "tenant:tenant-1:vector_space_estimate_watermark"
+ redis.setex(watermark_key, 123, raw_state)
+
+ with pytest.raises(VectorSpaceAdmissionError) as exc_info:
+ _check_estimate(sqlite_session, CloudPlan.SANDBOX, 10, redis=redis)
+
+ message = "Unable to reserve estimated vector storage right now. Please try again later."
+ assert str(exc_info.value) == message
+ assert exc_info.value.description == message
+ assert redis.values == {watermark_key: raw_state}
+ assert redis.ttls == {watermark_key: 123}
+
+
+ ("watermark", "expected_mb", "expected_document_ids"),
+ [
+ pytest.param(
+ {"projected_usage_bytes": 0, "document_ids": []},
+ 10,
+ ["文档-1"],
+ id="zero-usage",
+ ),
+ pytest.param(
+ {"projected_usage_bytes": 40 * _MEBIBYTE, "document_ids": ["document-0"], "extra": "ignored"},
+ 50,
+ ["document-0", "文档-1"],
+ id="extra-fields",
+ ),
+ pytest.param(
+ {"projected_usage_bytes": 50 * _MEBIBYTE, "document_ids": ["文档-1", "文档-1"]},
+ 50,
+ ["文档-1"],
+ id="duplicate-unicode-document-ids",
+ ),
+ ],
+)
+def test_valid_watermark_preserves_usage_and_document_ids(
+ sqlite_session: Session,
+ watermark: dict[str, object],
+ expected_mb: int,
+ expected_document_ids: list[str],
+) -> None:
+ redis = _FakeRedis()
+ watermark_key = "tenant:tenant-1:vector_space_estimate_watermark"
+ redis.setex(watermark_key, 123, json.dumps(watermark, ensure_ascii=False))
+
+ _check_estimate(sqlite_session, CloudPlan.SANDBOX, 10, document_id="文档-1", redis=redis)
+
+ assert json.loads(redis.values[watermark_key]) == {
+ "projected_usage_bytes": expected_mb * _MEBIBYTE,
+ "document_ids": expected_document_ids,
+ }
+ assert redis.ttls[watermark_key] == 1800
+
+
def test_fresh_usage_above_watermark_becomes_next_projection_base(sqlite_session: Session) -> None:
redis = _FakeRedis()
_check_estimate(