Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

chore: dep inject for model in website crawl api (#40156)

Likalikali3e8c0fdcef32
diff --git a/api/controllers/console/datasets/website.py b/api/controllers/console/datasets/website.py
index 39be3f3ce5f..9411d128195 100644
--- a/api/controllers/console/datasets/website.py
+++ b/api/controllers/console/datasets/website.py
@@ -1,13 +1,12 @@
from typing import Any, Literal
-from flask import request
from flask_restx import Resource
from pydantic import BaseModel, RootModel
from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
from controllers.console import console_ns
from controllers.console.datasets.error import WebsiteCrawlError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.wraps import account_initialization_required, model_validate, setup_required
from libs.login import login_required
from services.website_service import WebsiteCrawlApiRequest, WebsiteCrawlStatusApiRequest, WebsiteService
@@ -40,12 +39,11 @@ class WebsiteCrawlApi(Resource):
@setup_required
@login_required
@account_initialization_required
- def post(self):
- payload = WebsiteCrawlPayload.model_validate(console_ns.payload or {})
-
+ @model_validate(WebsiteCrawlPayload)
+ def post(self, req_data: WebsiteCrawlPayload):
# Create typed request and validate
try:
- api_request = WebsiteCrawlApiRequest.from_args(payload.model_dump())
+ api_request = WebsiteCrawlApiRequest.from_args(req_data.model_dump())
except ValueError as e:
raise WebsiteCrawlError(str(e))
@@ -69,12 +67,11 @@ class WebsiteCrawlStatusApi(Resource):
@setup_required
@login_required
@account_initialization_required
- def get(self, job_id: str):
- args = WebsiteCrawlStatusQuery.model_validate(request.args.to_dict())
-
+ @model_validate(WebsiteCrawlStatusQuery)
+ def get(self, req_data: WebsiteCrawlStatusQuery, job_id: str):
# Create typed request and validate
try:
- api_request = WebsiteCrawlStatusApiRequest.from_args(args.model_dump(), job_id)
+ api_request = WebsiteCrawlStatusApiRequest.from_args(req_data.model_dump(), job_id)
except ValueError as e:
raise WebsiteCrawlError(str(e))
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_website.py b/api/tests/unit_tests/controllers/console/datasets/test_website.py
index 5c7b857c20e..b790059f0c7 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_website.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_website.py
@@ -1,14 +1,15 @@
-from unittest.mock import Mock, PropertyMock, patch
+from unittest.mock import Mock
import pytest
from flask import Flask
from pytest_mock import MockerFixture
-from controllers.console import console_ns
from controllers.console.datasets.error import WebsiteCrawlError
from controllers.console.datasets.website import (
WebsiteCrawlApi,
+ WebsiteCrawlPayload,
WebsiteCrawlStatusApi,
+ WebsiteCrawlStatusQuery,
)
from services.website_service import (
WebsiteCrawlApiRequest,
@@ -58,16 +59,9 @@ class TestWebsiteCrawlApi:
"url": "https://example.com",
"options": {"depth": 1},
}
+ req_data = WebsiteCrawlPayload.model_validate(payload)
- with (
- app.test_request_context("/", json=payload),
- patch.object(
- type(console_ns),
- "payload",
- new_callable=PropertyMock,
- return_value=payload,
- ),
- ):
+ with app.test_request_context("/", json=payload):
mock_request = Mock(spec=WebsiteCrawlApiRequest)
mocker.patch.object(
WebsiteCrawlApiRequest,
@@ -81,7 +75,7 @@ class TestWebsiteCrawlApi:
return_value={"job_id": "job-1"},
)
- result, status = method(api)
+ result, status = method(api, req_data)
assert status == 200
assert result["job_id"] == "job-1"
@@ -95,16 +89,9 @@ class TestWebsiteCrawlApi:
"url": "bad-url",
"options": {},
}
+ req_data = WebsiteCrawlPayload.model_validate(payload)
- with (
- app.test_request_context("/", json=payload),
- patch.object(
- type(console_ns),
- "payload",
- new_callable=PropertyMock,
- return_value=payload,
- ),
- ):
+ with app.test_request_context("/", json=payload):
mocker.patch.object(
WebsiteCrawlApiRequest,
"from_args",
@@ -112,7 +99,7 @@ class TestWebsiteCrawlApi:
)
with pytest.raises(WebsiteCrawlError, match="invalid payload"):
- method(api)
+ method(api, req_data)
def test_crawl_service_error(self, app: Flask, mocker: MockerFixture):
api = WebsiteCrawlApi()
@@ -123,16 +110,9 @@ class TestWebsiteCrawlApi:
"url": "https://example.com",
"options": {},
}
+ req_data = WebsiteCrawlPayload.model_validate(payload)
- with (
- app.test_request_context("/", json=payload),
- patch.object(
- type(console_ns),
- "payload",
- new_callable=PropertyMock,
- return_value=payload,
- ),
- ):
+ with app.test_request_context("/", json=payload):
mock_request = Mock(spec=WebsiteCrawlApiRequest)
mocker.patch.object(
WebsiteCrawlApiRequest,
@@ -147,7 +127,7 @@ class TestWebsiteCrawlApi:
)
with pytest.raises(WebsiteCrawlError, match="crawl failed"):
- method(api)
+ method(api, req_data)
class TestWebsiteCrawlStatusApi:
@@ -156,14 +136,9 @@ class TestWebsiteCrawlStatusApi:
method = unwrap(api.get)
job_id = "job-123"
- args = {"provider": "firecrawl"}
+ req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
with app.test_request_context("/?provider=firecrawl"):
- mocker.patch(
- "controllers.console.datasets.website.request.args.to_dict",
- return_value=args,
- )
-
mock_request = Mock(spec=WebsiteCrawlStatusApiRequest)
mocker.patch.object(
WebsiteCrawlStatusApiRequest,
@@ -177,7 +152,7 @@ class TestWebsiteCrawlStatusApi:
return_value={"status": "completed"},
)
- result, status = method(api, job_id)
+ result, status = method(api, req_data, job_id)
assert status == 200
assert result["status"] == "completed"
@@ -187,14 +162,9 @@ class TestWebsiteCrawlStatusApi:
method = unwrap(api.get)
job_id = "job-123"
- args = {"provider": "firecrawl"}
+ req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
with app.test_request_context("/?provider=firecrawl"):
- mocker.patch(
- "controllers.console.datasets.website.request.args.to_dict",
- return_value=args,
- )
-
mocker.patch.object(
WebsiteCrawlStatusApiRequest,
"from_args",
@@ -202,21 +172,16 @@ class TestWebsiteCrawlStatusApi:
)
with pytest.raises(WebsiteCrawlError, match="invalid provider"):
- method(api, job_id)
+ method(api, req_data, job_id)
def test_get_status_service_error(self, app: Flask, mocker: MockerFixture):
api = WebsiteCrawlStatusApi()
method = unwrap(api.get)
job_id = "job-123"
- args = {"provider": "firecrawl"}
+ req_data = WebsiteCrawlStatusQuery.model_validate({"provider": "firecrawl"})
with app.test_request_context("/?provider=firecrawl"):
- mocker.patch(
- "controllers.console.datasets.website.request.args.to_dict",
- return_value=args,
- )
-
mock_request = Mock(spec=WebsiteCrawlStatusApiRequest)
mocker.patch.object(
WebsiteCrawlStatusApiRequest,
@@ -231,4 +196,4 @@ class TestWebsiteCrawlStatusApi:
)
with pytest.raises(WebsiteCrawlError, match="status lookup failed"):
- method(api, job_id)
+ method(api, req_data, job_id)