Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(import): preserve GitHub blob URL path components (#44280)

Winorenae3798420b31
diff --git a/api/libs/url_utils.py b/api/libs/url_utils.py
index adcac3add0e..59a9f8fbce2 100644
--- a/api/libs/url_utils.py
+++ b/api/libs/url_utils.py
@@ -1,3 +1,23 @@
+from urllib.parse import urlsplit, urlunsplit
+
+
def normalize_api_base_url(base_url: str) -> str:
"""Normalize a base URL to always end with /v1, avoiding double /v1 suffixes."""
return base_url.rstrip("/").removesuffix("/v1").rstrip("/") + "/v1"
+
+
+def github_blob_to_raw_url(url: str) -> str:
+ """Convert a GitHub file-view URL, preserving repository, file, query and fragment parts."""
+ parsed = urlsplit(url)
+ parts = parsed.path.split("/", 4)
+ if (
+ parsed.scheme != "https"
+ or parsed.netloc != "github.com"
+ or len(parts) != 5
+ or not parts[1]
+ or not parts[2]
+ or parts[3] != "blob"
+ or not parts[4]
+ ):
+ return url
+ return urlunsplit(parsed._replace(netloc="raw.githubusercontent.com", path=f"/{parts[1]}/{parts[2]}/{parts[4]}"))
diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py
index aa4d815d2ff..f1bf6da8932 100644
--- a/api/services/app_dsl_service.py
+++ b/api/services/app_dsl_service.py
@@ -50,6 +50,7 @@ from graphon.nodes.parameter_extractor.entities import ParameterExtractorNodeDat
from graphon.nodes.question_classifier.entities import QuestionClassifierNodeData
from graphon.nodes.tool.entities import ToolNodeData, ToolProviderType
from libs.datetime_utils import naive_utc_now
+from libs.url_utils import github_blob_to_raw_url
from models import Account, App, AppMode
from models.agent import AgentScope, WorkflowAgentBindingType, WorkflowAgentNodeBinding
from models.agent_config_entities import WorkflowNodeJobConfig
@@ -227,14 +228,8 @@ class AppDslService:
)
try:
parsed_url = urlparse(yaml_url)
- if (
- parsed_url.scheme == "https"
- and parsed_url.netloc == "github.com"
- and parsed_url.path.endswith((".yml", ".yaml"))
- and "/blob/" in parsed_url.path
- ):
- yaml_url = yaml_url.replace("https://github.com", "https://raw.githubusercontent.com")
- yaml_url = yaml_url.replace("/blob/", "/")
+ if parsed_url.path.endswith((".yml", ".yaml")):
+ yaml_url = github_blob_to_raw_url(yaml_url)
response = remote_fetcher.make_request("GET", yaml_url.strip(), follow_redirects=True, timeout=(10, 10))
response.raise_for_status()
raw_content = response.content
diff --git a/api/services/app_import_source.py b/api/services/app_import_source.py
index 1508c8c3c96..810b0a19b0a 100644
--- a/api/services/app_import_source.py
+++ b/api/services/app_import_source.py
@@ -4,7 +4,7 @@ import tempfile
from collections.abc import Generator
from contextlib import contextmanager
from typing import BinaryIO, cast
-from urllib.parse import urlsplit, urlunsplit
+from urllib.parse import urlsplit
import httpx
import yaml
@@ -12,6 +12,7 @@ import yaml
from configs import dify_config
from core.file import remote_fetcher
from core.tools.errors import ToolSSRFError
+from libs.url_utils import github_blob_to_raw_url
from services.agent.errors import InvalidRosterAgentPackageError, RosterAgentPackageTooLargeError
from services.dsl_content import DSL_MAX_SIZE
@@ -29,10 +30,7 @@ def download_app_import_source(url: str, *, max_bytes: int | None = None) -> Gen
raise ValueError("An absolute HTTP(S) URL is required")
except ValueError as exc:
raise InvalidRosterAgentPackageError("App import URL is invalid") from exc
- if parsed.scheme == "https" and parsed.netloc == "github.com" and "/blob/" in parsed.path:
- url = urlunsplit(
- parsed._replace(netloc="raw.githubusercontent.com", path=parsed.path.replace("/blob/", "/", 1))
- )
+ url = github_blob_to_raw_url(url)
with tempfile.SpooledTemporaryFile(max_size=16 * 1024 * 1024, mode="w+b") as source:
try:
diff --git a/api/services/rag_pipeline/rag_pipeline_dsl_service.py b/api/services/rag_pipeline/rag_pipeline_dsl_service.py
index 04f1ddb4d7d..a4e854a1c1e 100644
--- a/api/services/rag_pipeline/rag_pipeline_dsl_service.py
+++ b/api/services/rag_pipeline/rag_pipeline_dsl_service.py
@@ -38,6 +38,7 @@ from graphon.nodes.llm.entities import LLMNodeData, ModelConfig
from graphon.nodes.parameter_extractor.entities import ParameterExtractorNodeData
from graphon.nodes.question_classifier.entities import QuestionClassifierNodeData
from graphon.nodes.tool.entities import ToolNodeData
+from libs.url_utils import github_blob_to_raw_url
from models import Account
from models.dataset import Dataset, DatasetCollectionBinding, Pipeline
from models.enums import CollectionBindingType, DatasetRuntimeMode
@@ -127,13 +128,8 @@ class RagPipelineDslService:
)
try:
parsed_url = urlparse(yaml_url)
- if (
- parsed_url.scheme == "https"
- and parsed_url.netloc == "github.com"
- and parsed_url.path.endswith((".yml", ".yaml"))
- ):
- yaml_url = yaml_url.replace("https://github.com", "https://raw.githubusercontent.com")
- yaml_url = yaml_url.replace("/blob/", "/")
+ if parsed_url.path.endswith((".yml", ".yaml")):
+ yaml_url = github_blob_to_raw_url(yaml_url)
response = remote_fetcher.make_request("GET", yaml_url.strip(), follow_redirects=True, timeout=(10, 10))
response.raise_for_status()
raw_content = response.content
diff --git a/api/services/workflow/dsl_import.py b/api/services/workflow/dsl_import.py
index 1e22833ecd1..50019231fdc 100644
--- a/api/services/workflow/dsl_import.py
+++ b/api/services/workflow/dsl_import.py
@@ -19,6 +19,7 @@ from core.file import remote_fetcher
from factories import variable_factory
from graphon.variables import SecretVariable, SegmentType, VariableBase
from graphon.variables.exc import VariableError
+from libs.url_utils import github_blob_to_raw_url
from machinery.context import RequestContext
from models import AppMode, Workflow
from services.dsl_content import DSL_MAX_SIZE, dsl_content_size
@@ -77,14 +78,8 @@ def fetch_yaml_url(yaml_url: str) -> str:
"""The YAML text at a URL, fetched as AppDslService.import_app does; a GitHub blob link reads its raw file."""
try:
parsed_url = urlparse(yaml_url)
- if (
- parsed_url.scheme == "https"
- and parsed_url.netloc == "github.com"
- and parsed_url.path.endswith((".yml", ".yaml"))
- and "/blob/" in parsed_url.path
- ):
- yaml_url = yaml_url.replace("https://github.com", "https://raw.githubusercontent.com")
- yaml_url = yaml_url.replace("/blob/", "/")
+ if parsed_url.path.endswith((".yml", ".yaml")):
+ yaml_url = github_blob_to_raw_url(yaml_url)
response = remote_fetcher.make_request("GET", yaml_url.strip(), follow_redirects=True, timeout=(10, 10))
response.raise_for_status()
raw_content = response.content
diff --git a/api/tests/unit_tests/services/test_app_import_source.py b/api/tests/unit_tests/services/test_app_import_source.py
index 7d41932c3f7..094bc198c06 100644
--- a/api/tests/unit_tests/services/test_app_import_source.py
+++ b/api/tests/unit_tests/services/test_app_import_source.py
@@ -14,10 +14,26 @@ from services.agent.errors import InvalidRosterAgentPackageError, RosterAgentPac
("url", "download_url"),
[
(" https://example.com/agent.IFPKG?token=secret ", "https://example.com/agent.IFPKG?token=secret"),
+ (
+ "https://github.com/user-attachments/files/123/flow.yaml",
+ "https://github.com/user-attachments/files/123/flow.yaml",
+ ),
(
"https://github.com/org/repo/blob/main/agent.ifpkg",
"https://raw.githubusercontent.com/org/repo/main/agent.ifpkg",
),
+ (
+ "https://github.com/org/repo/blob/main/templates/blob/agent.ifpkg?source=/blob/file",
+ "https://raw.githubusercontent.com/org/repo/main/templates/blob/agent.ifpkg?source=/blob/file",
+ ),
+ (
+ "https://github.com/blob/repo/blob/main/agent.ifpkg",
+ "https://raw.githubusercontent.com/blob/repo/main/agent.ifpkg",
+ ),
+ (
+ "https://github.com/org/repo/raw/main/templates/blob/agent.ifpkg",
+ "https://github.com/org/repo/raw/main/templates/blob/agent.ifpkg",
+ ),
],
)
def test_download_preserves_binary_and_closes_resources(
diff --git a/api/tests/unit_tests/services/test_github_import_urls.py b/api/tests/unit_tests/services/test_github_import_urls.py
new file mode 100644
index 00000000000..c6cbd496fd5
--- /dev/null
+++ b/api/tests/unit_tests/services/test_github_import_urls.py
@@ -0,0 +1,59 @@
+"""Check URL selection at the legacy YAML import download boundary."""
+
+import httpx
+import pytest
+from sqlalchemy.orm import Session
+
+from core.file import remote_fetcher
+from models import Account
+from services.app_dsl_service import AppDslService
+from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelineDslService
+
+
[email protected]("importer", ["app", "rag-pipeline"])
+ ("url", "download_url"),
+ [
+ ("https://example.com/download", "https://example.com/download"),
+ (
+ "https://github.com/org/repo/blob/main/templates/blob/flow.yaml?source=/blob/file",
+ "https://raw.githubusercontent.com/org/repo/main/templates/blob/flow.yaml?source=/blob/file",
+ ),
+ (
+ "https://github.com/blob/repo/blob/main/flow.yml",
+ "https://raw.githubusercontent.com/blob/repo/main/flow.yml",
+ ),
+ (
+ "https://github.com/org/repo/raw/main/templates/blob/flow.yaml",
+ "https://github.com/org/repo/raw/main/templates/blob/flow.yaml",
+ ),
+ (
+ "https://github.com/user-attachments/files/123/flow.yaml",
+ "https://github.com/user-attachments/files/123/flow.yaml",
+ ),
+ ],
+)
+def test_yaml_import_preserves_github_url_components(
+ monkeypatch: pytest.MonkeyPatch, unbound_session: Session, importer: str, url: str, download_url: str
+) -> None:
+ requested: list[str] = []
+
+ def fetch(method: str, url: str, **_kwargs: object) -> httpx.Response:
+ requested.append(url)
+ # An empty successful response stops after URL selection, before resource creation.
+ return httpx.Response(200, request=httpx.Request(method, url), content=b"")
+
+ monkeypatch.setattr(remote_fetcher, "make_request", fetch)
+ account = Account(name="Test Account", email="[email protected]")
+ if importer == "app":
+ result = AppDslService(session=unbound_session).import_app(
+ account=account, import_mode="yaml-url", yaml_url=url
+ )
+ else:
+ result = RagPipelineDslService(session=unbound_session).import_rag_pipeline(
+ account=account, import_mode="yaml-url", yaml_url=url
+ )
+
+ assert requested == [download_url]
+ assert result.error == "Empty content from url"
+ assert not unbound_session.in_transaction()
diff --git a/api/tests/unit_tests/services/workflow/test_dsl_import.py b/api/tests/unit_tests/services/workflow/test_dsl_import.py
index 40bc52c4d9e..fba09e4c01c 100644
--- a/api/tests/unit_tests/services/workflow/test_dsl_import.py
+++ b/api/tests/unit_tests/services/workflow/test_dsl_import.py
@@ -1,6 +1,7 @@
from collections.abc import Generator, Mapping
from contextlib import contextmanager
+import httpx
import pytest
import yaml
@@ -172,3 +173,47 @@ def test_prepare_hands_an_oversized_dsl_to_the_import_unchanged() -> None:
source = _padded(_dsl("workflow", _START, _END))
assert _prepare(_Store(), source=source, app_id=None) == source
+
+
+ ("url", "download_url"),
+ [
+ ("https://example.com/download", "https://example.com/download"),
+ (
+ "https://github.com/org/repo/blob/main/templates/blob/flow.yaml?source=/blob/file",
+ "https://raw.githubusercontent.com/org/repo/main/templates/blob/flow.yaml?source=/blob/file",
+ ),
+ (
+ "https://github.com/blob/repo/blob/main/flow.yml",
+ "https://raw.githubusercontent.com/blob/repo/main/flow.yml",
+ ),
+ (
+ "https://github.com/org/repo/raw/main/templates/blob/flow.yaml",
+ "https://github.com/org/repo/raw/main/templates/blob/flow.yaml",
+ ),
+ ("https://example.com/templates/blob/flow.yaml", "https://example.com/templates/blob/flow.yaml"),
+ (
+ "https://github.com/user-attachments/files/123/flow.yaml",
+ "https://github.com/user-attachments/files/123/flow.yaml",
+ ),
+ ],
+)
+def test_prepare_preserves_github_blob_path_components(
+ monkeypatch: pytest.MonkeyPatch, url: str, download_url: str
+) -> None:
+ requested: list[str] = []
+
+ def fetch(method: str, url: str, **_kwargs: object) -> httpx.Response:
+ requested.append(url)
+ return httpx.Response(
+ 200 if url == download_url else 404,
+ request=httpx.Request(method, url),
+ content=_dsl("workflow", _START, _END).encode(),
+ )
+
+ monkeypatch.setattr(dsl_import.remote_fetcher, "make_request", fetch)
+ prepared = yaml.safe_load(_prepare(_Store(_TARGET), source=url, draft_hash="token-1"))
+
+ assert requested == [download_url]
+ assert prepared["app"]["mode"] == "workflow"
+ assert [node["id"] for node in prepared["workflow"]["graph"]["nodes"]] == ["start", "end"]