Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(api): replace json.loads with Pydantic validation in repositories and tasks (#43704)
haloneko1e3fe6ba9396
diff --git a/api/repositories/data_source/credential_repository.py b/api/repositories/data_source/credential_repository.pyindex 9862ada014a..3d818301808 100644--- a/api/repositories/data_source/credential_repository.py+++ b/api/repositories/data_source/credential_repository.py@@ -1,6 +1,5 @@"""Actor-aware and trusted-source SQLAlchemy repository for datasource credentials."""-import jsonfrom collections.abc import Mappingfrom typing import cast@@ -22,7 +21,7 @@ _CREDENTIALS_ADAPTER = TypeAdapter(dict[str, object])def _source_mapping(value: object) -> Mapping[str, object]:try:if isinstance(value, str):- return _CREDENTIALS_ADAPTER.validate_python(json.loads(value))+ return _CREDENTIALS_ADAPTER.validate_json(value)return _CREDENTIALS_ADAPTER.validate_python(value)except (TypeError, ValueError, ValidationError):return {}diff --git a/api/repositories/knowledge/document_repository.py b/api/repositories/knowledge/document_repository.pyindex 55ba407c8f3..d5d9f8724f3 100644--- a/api/repositories/knowledge/document_repository.py+++ b/api/repositories/knowledge/document_repository.py@@ -1,6 +1,5 @@"""SQLAlchemy repository for tenant-owned document state."""-import jsonfrom collections.abc import Mappingfrom pydantic import TypeAdapter, ValidationError@@ -26,7 +25,7 @@ def _mapping(value: object) -> Mapping[str, object] | None:return Nonetry:if isinstance(value, str):- return _MAPPING_ADAPTER.validate_python(json.loads(value))+ return _MAPPING_ADAPTER.validate_json(value)return _MAPPING_ADAPTER.validate_python(value)except (TypeError, ValueError, ValidationError):return {}diff --git a/api/repositories/upload_file_delivery_repository.py b/api/repositories/upload_file_delivery_repository.pyindex 13a2a3518e9..ca088895dd5 100644--- a/api/repositories/upload_file_delivery_repository.py+++ b/api/repositories/upload_file_delivery_repository.py@@ -1,8 +1,8 @@"""SQLAlchemy query adapter for public UploadFile delivery endpoints."""-import json-from typing import cast, override+from typing import override+from pydantic import TypeAdapterfrom sqlalchemy import Row, Select, selectfrom sqlalchemy.orm import Session, sessionmaker@@ -14,6 +14,8 @@ from services.upload_file_delivery_service import (UploadFileDeliveryRecord,)+_TENANT_CUSTOM_CONFIG_ADAPTER = TypeAdapter(TenantCustomConfigDict)+class UploadFileDeliveryQueryRepository(UploadFileDeliveryQuery):def __init__(self, *, session_factory: sessionmaker[Session]) -> None:@@ -36,7 +38,7 @@ class UploadFileDeliveryQueryRepository(UploadFileDeliveryQuery):raise UploadFileDeliveryNotFoundErrorcustom_config = (- cast(TenantCustomConfigDict, json.loads(workspace_row.custom_config))+ _TENANT_CUSTOM_CONFIG_ADAPTER.validate_json(workspace_row.custom_config)if workspace_row.custom_configelse {})diff --git a/api/tasks/enterprise_telemetry_task.py b/api/tasks/enterprise_telemetry_task.pyindex 7d5ea7c0a5a..bdc2fe9ee7d 100644--- a/api/tasks/enterprise_telemetry_task.py+++ b/api/tasks/enterprise_telemetry_task.py@@ -5,7 +5,6 @@ from the enterprise_telemetry queue. It deserializes envelopes anddispatches them to the EnterpriseMetricHandler."""-import jsonimport loggingfrom celery import shared_task@@ -31,8 +30,7 @@ def process_enterprise_telemetry(envelope_json: str) -> None:"""try:# Deserialize envelope- envelope_dict = json.loads(envelope_json)- envelope = TelemetryEnvelope.model_validate(envelope_dict)+ envelope = TelemetryEnvelope.model_validate_json(envelope_json)# Process through handlerhandler = EnterpriseMetricHandler()diff --git a/api/tasks/mail_human_input_delivery_task.py b/api/tasks/mail_human_input_delivery_task.pyindex 8ed50071df5..6748ca3ae8e 100644--- a/api/tasks/mail_human_input_delivery_task.py+++ b/api/tasks/mail_human_input_delivery_task.py@@ -1,11 +1,10 @@-import jsonimport loggingimport timefrom dataclasses import dataclass-from typing import Anyimport clickfrom celery import shared_task+from pydantic import BaseModel, ConfigDictfrom sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmaker@@ -48,14 +47,23 @@ def _build_form_link(token: str) -> str:return f"{base_url.rstrip('/')}/form/{token}"+class _RecipientPayload(BaseModel):+ """Shape of HumanInputFormRecipient.recipient_payload written by the recipients layer."""++ model_config = ConfigDict(extra="ignore")++ email: str+ TYPE: RecipientType++def _parse_recipient_payload(payload: str) -> tuple[str | None, RecipientType | None]:try:- payload_dict: dict[str, Any] = json.loads(payload)+ recipient = _RecipientPayload.model_validate_json(payload)except Exception:logger.exception("Failed to parse recipient payload")return None, None- return payload_dict.get("email"), payload_dict.get("TYPE")+ return recipient.email, recipient.TYPEdef _load_email_jobs(session: Session, form: HumanInputForm) -> list[_EmailDeliveryJob]:diff --git a/api/tasks/process_tenant_plugin_autoupgrade_check_task.py b/api/tasks/process_tenant_plugin_autoupgrade_check_task.pyindex 8457117b757..d50dc6b69ef 100644--- a/api/tasks/process_tenant_plugin_autoupgrade_check_task.py+++ b/api/tasks/process_tenant_plugin_autoupgrade_check_task.py@@ -5,6 +5,7 @@ import typingimport clickfrom celery import shared_task+from pydantic import TypeAdapterfrom core.helper.marketplace import batch_fetch_plugin_manifests, get_plugin_pkg_urlfrom core.plugin.entities.marketplace import MarketplacePluginDeclaration, MarketplacePluginSnapshot@@ -25,6 +26,9 @@ RETRY_TIMES_OF_ONE_PLUGIN_IN_ONE_TENANT = 3CACHE_REDIS_KEY_PREFIX = "plugin_autoupgrade_check_task:cached_plugin_snapshot:"CACHE_REDIS_TTL = 60 * 60 # 1 hour+# A cached miss is stored as JSON null, so None has to stay a valid payload.+_CACHED_MANIFEST_ADAPTER = TypeAdapter(typing.Union[MarketplacePluginSnapshot, None])+def _get_redis_cache_key(plugin_id: str) -> str:"""Generate Redis cache key for plugin manifest."""@@ -45,11 +49,7 @@ def _get_cached_manifest(plugin_id: str) -> typing.Union[MarketplacePluginSnapshif cached_data is None:return False- cached_json = json.loads(cached_data)- if cached_json is None:- return None-- return MarketplacePluginSnapshot.model_validate(cached_json)+ return _CACHED_MANIFEST_ADAPTER.validate_json(cached_data)except Exception:logger.exception("Failed to get cached manifest for plugin %s", plugin_id)return Falsediff --git a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.pyindex c76edab1b4f..1c5716dcec8 100644--- a/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py+++ b/api/tasks/rag_pipeline/priority_rag_pipeline_run_task.py@@ -1,5 +1,4 @@import contextvars-import jsonimport loggingimport timeimport uuid@@ -10,6 +9,7 @@ from typing import Anyimport clickfrom celery import shared_task # type: ignorefrom flask import current_app, g+from pydantic import TypeAdapterfrom sqlalchemy import selectfrom sqlalchemy.orm import Session, sessionmaker@@ -31,6 +31,9 @@ from services.file_service import FileServicelogger = logging.getLogger(__name__)+# Each entry is validated into RagPipelineInvokeEntity inside the worker.+_INVOKE_ENTITIES_ADAPTER = TypeAdapter(list[dict[str, Any]])+@shared_task(queue="priority_pipeline")def priority_rag_pipeline_run_task(@@ -50,7 +53,7 @@ def priority_rag_pipeline_run_task(rag_pipeline_invoke_entities_content = FileService(db.engine).get_file_content(rag_pipeline_invoke_entities_file_id)- rag_pipeline_invoke_entities = json.loads(rag_pipeline_invoke_entities_content)+ rag_pipeline_invoke_entities = _INVOKE_ENTITIES_ADAPTER.validate_json(rag_pipeline_invoke_entities_content)logger.info("tenant %s received %d rag pipeline invoke entities", tenant_id, len(rag_pipeline_invoke_entities))diff --git a/api/tasks/rag_pipeline/rag_pipeline_run_task.py b/api/tasks/rag_pipeline/rag_pipeline_run_task.pyindex 9774be64e9f..d164e4fec97 100644--- a/api/tasks/rag_pipeline/rag_pipeline_run_task.py+++ b/api/tasks/rag_pipeline/rag_pipeline_run_task.py@@ -1,5 +1,4 @@import contextvars-import jsonimport loggingimport timeimport uuid@@ -11,6 +10,7 @@ from typing import Anyimport clickfrom celery import group, shared_taskfrom flask import current_app, g+from pydantic import TypeAdapterfrom sqlalchemy import selectfrom sqlalchemy.orm import sessionmaker@@ -32,6 +32,9 @@ from services.file_service import FileServicelogger = logging.getLogger(__name__)+# Each entry is validated into RagPipelineInvokeEntity inside the worker.+_INVOKE_ENTITIES_ADAPTER = TypeAdapter(list[dict[str, Any]])+def chunked(iterable: Sequence, size: int):it = iter(iterable)@@ -56,7 +59,7 @@ def rag_pipeline_run_task(rag_pipeline_invoke_entities_content = FileService(db.engine).get_file_content(rag_pipeline_invoke_entities_file_id)- rag_pipeline_invoke_entities = json.loads(rag_pipeline_invoke_entities_content)+ rag_pipeline_invoke_entities = _INVOKE_ENTITIES_ADAPTER.validate_json(rag_pipeline_invoke_entities_content)logger.info("tenant %s received %d rag pipeline invoke entities", tenant_id, len(rag_pipeline_invoke_entities))diff --git a/api/tests/unit_tests/tasks/test_mail_human_input_delivery_task.py b/api/tests/unit_tests/tasks/test_mail_human_input_delivery_task.pyindex 60c047ad20d..4877ea30283 100644--- a/api/tests/unit_tests/tasks/test_mail_human_input_delivery_task.py+++ b/api/tests/unit_tests/tasks/test_mail_human_input_delivery_task.py@@ -1,3 +1,4 @@+import jsonfrom collections.abc import Sequencefrom datetime import UTC, datetime, timedeltafrom types import SimpleNamespace@@ -7,7 +8,7 @@ import pytestfrom sqlalchemy.engine import Enginefrom sqlalchemy.orm import Session, sessionmaker-from models.human_input import HumanInputForm+from models.human_input import HumanInputForm, RecipientTypefrom tasks import mail_human_input_delivery_task as task_module@@ -178,3 +179,23 @@ def test_dispatch_human_input_email_task_sanitizes_subject()assert mail.sent[0]["subject"] == "Notice BCC:[email protected] Alert"+++def test_parse_recipient_payload_reads_email_and_type():+ payload = json.dumps({"email": "[email protected]", "TYPE": RecipientType.EMAIL_MEMBER.value})++ assert task_module._parse_recipient_payload(payload) == ("[email protected]", RecipientType.EMAIL_MEMBER)+++ "payload",+ [+ "not json",+ json.dumps({"TYPE": RecipientType.EMAIL_MEMBER.value}),+ json.dumps({"email": "[email protected]", "TYPE": "unknown-channel"}),+ json.dumps({"email": 42, "TYPE": RecipientType.EMAIL_MEMBER.value}),+ ],+)+def test_parse_recipient_payload_rejects_unusable_payloads(payload: str):+ """Payloads the recipients layer cannot produce must degrade to a skipped recipient."""+ assert task_module._parse_recipient_payload(payload) == (None, None)