Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

refactor(api): consolidate data migration CLI commands (#43483)

非法操作c60b3962df79
diff --git a/api/commands/__init__.py b/api/commands/__init__.py
index 14bffa072e3..c5c9be55bcb 100644
--- a/api/commands/__init__.py
+++ b/api/commands/__init__.py
@@ -3,11 +3,12 @@ CLI command modules extracted from `commands.py`.
"""
from .account import create_tenant, reset_email, reset_password
-from .data_migrate import data_migrate, legacy_model_types
from .data_migration import (
+ data_migrate,
export_migration_data,
export_migration_data_template,
import_migration_data,
+ legacy_model_types,
migration_data_wizard,
)
from .plugin import (
diff --git a/api/commands/data_migrate.py b/api/commands/data_migrate.py
deleted file mode 100644
index f1a7299b5c3..00000000000
--- a/api/commands/data_migrate.py
+++ /dev/null
@@ -1,186 +0,0 @@
-import io
-import os
-import sys
-from contextlib import AbstractContextManager, nullcontext
-from pathlib import Path
-from typing import cast
-
-import click
-
-from commands.rbac import (
- migrate_dataset_permissions_to_rbac,
- migrate_only_me_resource_whitelist_scopes_to_automatic_include,
- migrate_resource_whitelist_scopes_to_automatic_include, # noqa: F401
-)
-from extensions.ext_database import db
-from graphon.model_runtime.entities.model_entities import ModelType
-from services.legacy_model_type_migration import (
- VALID_TABLE_NAMES,
- LegacyModelTypeMigrationService,
- load_tenant_ids_from_file,
-)
-
-_SUPPORTED_MODEL_TYPE_CHOICES = (
- ModelType.LLM.value,
- ModelType.TEXT_EMBEDDING.value,
- ModelType.RERANK.value,
-)
-_DEFAULT_CONCURRENCY = os.cpu_count() or 1
-
-
-def _normalize_multi_value_option(
- values: tuple[str, ...],
- *,
- valid_values: tuple[str, ...],
- option_name: str,
-) -> tuple[str, ...]:
- normalized_values: list[str] = []
- seen_values: set[str] = set()
-
- for value in values:
- for item in value.split(","):
- normalized_item = item.strip()
- if not normalized_item:
- continue
- if normalized_item not in valid_values:
- raise click.BadParameter(
- f"invalid value '{normalized_item}'. valid values: {', '.join(valid_values)}",
- param_hint=option_name,
- )
- if normalized_item in seen_values:
- continue
- seen_values.add(normalized_item)
- normalized_values.append(normalized_item)
-
- return tuple(normalized_values)
-
-
- "data-migrate",
- help="Online data migration commands.",
-)
-def data_migrate() -> None:
- """Namespace for production data migration commands."""
-
-
- "legacy-model-types",
- help=(
- "Migrate legacy provider model_type values to canonical values. "
- "Default is dry-run and emits JSON lines only. "
- "If --tables includes provider_model_credentials, the command may also update "
- "provider_models and load_balancing_model_configs references so merged credentials stay reachable."
- ),
-)
- "--apply",
- is_flag=True,
- default=False,
- help="Apply the migration. Default is dry-run.",
-)
- "--tables",
- "tables",
- multiple=True,
- type=str,
- help=(
- "Limit migration to specific tables. Accepts comma-separated values or repeated flags.\n"
- "\n"
- "Options: load_balancing_model_configs, provider_model_credentials, "
- "provider_model_settings, provider_models, tenant_default_models.\n\n"
- "When provider_model_credentials is selected, provider_models and "
- "load_balancing_model_configs may also be updated for credential reference rewrites.\n"
- "\n"
- "If unspecified, all relevant tables are migrated."
- ),
-)
- "--model-types",
- "model_types",
- multiple=True,
- type=str,
- help=(
- "Canonical model types to migrate. Accepts comma-separated values or repeated flags.\n"
- "\n"
- "Options: llm,text-embedding,rerank\n"
- "\n"
- "If unspecified, all relevant legacy model types are migrated."
- ),
-)
- "--tenant-id-file",
- type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),
- help="Optional file containing tenant ids, one per line.",
-)
- "--output",
- type=click.Path(dir_okay=False, resolve_path=True, path_type=Path),
- help=(
- "Optional file path for JSON lines event logs. Defaults to stdout.\n"
- "It's highly recommended to save the event logs to a file and preserve it for a period of time."
- ),
-)
- "--concurrency",
- type=click.IntRange(min=1),
- default=_DEFAULT_CONCURRENCY,
- show_default=True,
- help="Number of tenant-level worker threads to run in parallel.",
-)
-def legacy_model_types(
- apply: bool,
- tables: tuple[str, ...],
- model_types: tuple[str, ...],
- tenant_id_file: str | None,
- output: Path | None,
- concurrency: int = _DEFAULT_CONCURRENCY,
-) -> None:
- """
- Migrate legacy provider-related model_type values and emit JSON lines events.
- """
-
- normalized_tables = _normalize_multi_value_option(
- tables,
- valid_values=VALID_TABLE_NAMES,
- option_name="--tables",
- )
- normalized_model_types = _normalize_multi_value_option(
- model_types,
- valid_values=_SUPPORTED_MODEL_TYPE_CHOICES,
- option_name="--model-types",
- )
- selected_model_types = (
- tuple(ModelType(model_type) for model_type in normalized_model_types)
- if normalized_model_types
- else (
- ModelType.LLM,
- ModelType.TEXT_EMBEDDING,
- ModelType.RERANK,
- )
- )
- tenant_ids = load_tenant_ids_from_file(tenant_id_file) if tenant_id_file else None
-
- output_context: AbstractContextManager[io.TextIOBase]
- if output is None:
- output_context = nullcontext(cast(io.TextIOBase, sys.stdout))
- else:
- try:
- output_context = output.open("w", encoding="utf-8")
- except OSError as exc:
- raise click.ClickException(f"failed to open output file '{output}': {exc.strerror or exc}") from exc
-
- with output_context as output_stream:
- LegacyModelTypeMigrationService(
- engine=db.engine,
- apply=apply,
- concurrency=concurrency,
- output=cast(io.TextIOBase, output_stream),
- tables=normalized_tables or None,
- model_types=selected_model_types,
- tenant_ids=tenant_ids,
- ).migrate()
-
-
-data_migrate.add_command(legacy_model_types)
-data_migrate.add_command(migrate_dataset_permissions_to_rbac)
-data_migrate.add_command(migrate_only_me_resource_whitelist_scopes_to_automatic_include)
diff --git a/api/commands/data_migration.py b/api/commands/data_migration.py
index 8c2627601a6..6bd36ee2200 100644
--- a/api/commands/data_migration.py
+++ b/api/commands/data_migration.py
@@ -1,6 +1,12 @@
+"""Online data migration and application import/export CLI commands."""
+
from __future__ import annotations
+import io
import json
+import os
+import sys
+from contextlib import AbstractContextManager, nullcontext
from datetime import datetime
from pathlib import Path
from typing import Any, cast
@@ -11,8 +17,14 @@ import sqlalchemy as sa
import yaml
from sqlalchemy.orm import Session
+from commands.rbac import (
+ migrate_dataset_permissions_to_rbac,
+ migrate_only_me_resource_whitelist_scopes_to_automatic_include,
+ migrate_resource_whitelist_scopes_to_automatic_include, # noqa: F401
+)
from core.db.session_factory import session_factory
from extensions.ext_database import db
+from graphon.model_runtime.entities.model_entities import ModelType
from models import Tenant
from models.model import App
from models.tools import ApiToolProvider, MCPToolProvider, WorkflowToolProvider
@@ -29,7 +41,18 @@ from services.data_migration.export_service import ExportConfigParser, Migration
from services.data_migration.import_service import ImportRequest, MigrationImportService
from services.data_migration.package_service import MigrationPackageService
from services.data_migration.report_service import MigrationReportService
+from services.legacy_model_type_migration import (
+ VALID_TABLE_NAMES,
+ LegacyModelTypeMigrationService,
+ load_tenant_ids_from_file,
+)
+_SUPPORTED_MODEL_TYPE_CHOICES = (
+ ModelType.LLM.value,
+ ModelType.TEXT_EMBEDDING.value,
+ ModelType.RERANK.value,
+)
+_DEFAULT_CONCURRENCY = os.cpu_count() or 1
ID_STRATEGY_CHOICES = ["preserve-id", "generate-new-id"]
CONFLICT_STRATEGY_CHOICES = ["fail", "skip", "update"]
SUPPORTED_WIZARD_APP_MODES = ["workflow", "advanced-chat"]
@@ -37,6 +60,164 @@ WizardToolMap = dict[str, dict[str, str | None]]
WizardToolSelection = dict[str, list[str]]
+def _normalize_multi_value_option(
+ values: tuple[str, ...],
+ *,
+ valid_values: tuple[str, ...],
+ option_name: str,
+) -> tuple[str, ...]:
+ normalized_values: list[str] = []
+ seen_values: set[str] = set()
+
+ for value in values:
+ for item in value.split(","):
+ normalized_item = item.strip()
+ if not normalized_item:
+ continue
+ if normalized_item not in valid_values:
+ raise click.BadParameter(
+ f"invalid value '{normalized_item}'. valid values: {', '.join(valid_values)}",
+ param_hint=option_name,
+ )
+ if normalized_item in seen_values:
+ continue
+ seen_values.add(normalized_item)
+ normalized_values.append(normalized_item)
+
+ return tuple(normalized_values)
+
+
+ "data-migrate",
+ help="Online data migration commands.",
+)
+def data_migrate() -> None:
+ """Namespace for production data migration commands."""
+
+
+ "legacy-model-types",
+ help=(
+ "Migrate legacy provider model_type values to canonical values. "
+ "Default is dry-run and emits JSON lines only. "
+ "If --tables includes provider_model_credentials, the command may also update "
+ "provider_models and load_balancing_model_configs references so merged credentials stay reachable."
+ ),
+)
+ "--apply",
+ is_flag=True,
+ default=False,
+ help="Apply the migration. Default is dry-run.",
+)
+ "--tables",
+ "tables",
+ multiple=True,
+ type=str,
+ help=(
+ "Limit migration to specific tables. Accepts comma-separated values or repeated flags.\n"
+ "\n"
+ "Options: load_balancing_model_configs, provider_model_credentials, "
+ "provider_model_settings, provider_models, tenant_default_models.\n\n"
+ "When provider_model_credentials is selected, provider_models and "
+ "load_balancing_model_configs may also be updated for credential reference rewrites.\n"
+ "\n"
+ "If unspecified, all relevant tables are migrated."
+ ),
+)
+ "--model-types",
+ "model_types",
+ multiple=True,
+ type=str,
+ help=(
+ "Canonical model types to migrate. Accepts comma-separated values or repeated flags.\n"
+ "\n"
+ "Options: llm,text-embedding,rerank\n"
+ "\n"
+ "If unspecified, all relevant legacy model types are migrated."
+ ),
+)
+ "--tenant-id-file",
+ type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),
+ help="Optional file containing tenant ids, one per line.",
+)
+ "--output",
+ type=click.Path(dir_okay=False, resolve_path=True, path_type=Path),
+ help=(
+ "Optional file path for JSON lines event logs. Defaults to stdout.\n"
+ "It's highly recommended to save the event logs to a file and preserve it for a period of time."
+ ),
+)
+ "--concurrency",
+ type=click.IntRange(min=1),
+ default=_DEFAULT_CONCURRENCY,
+ show_default=True,
+ help="Number of tenant-level worker threads to run in parallel.",
+)
+def legacy_model_types(
+ apply: bool,
+ tables: tuple[str, ...],
+ model_types: tuple[str, ...],
+ tenant_id_file: str | None,
+ output: Path | None,
+ concurrency: int = _DEFAULT_CONCURRENCY,
+) -> None:
+ """
+ Migrate legacy provider-related model_type values and emit JSON lines events.
+ """
+
+ normalized_tables = _normalize_multi_value_option(
+ tables,
+ valid_values=VALID_TABLE_NAMES,
+ option_name="--tables",
+ )
+ normalized_model_types = _normalize_multi_value_option(
+ model_types,
+ valid_values=_SUPPORTED_MODEL_TYPE_CHOICES,
+ option_name="--model-types",
+ )
+ selected_model_types = (
+ tuple(ModelType(model_type) for model_type in normalized_model_types)
+ if normalized_model_types
+ else (
+ ModelType.LLM,
+ ModelType.TEXT_EMBEDDING,
+ ModelType.RERANK,
+ )
+ )
+ tenant_ids = load_tenant_ids_from_file(tenant_id_file) if tenant_id_file else None
+
+ output_context: AbstractContextManager[io.TextIOBase]
+ if output is None:
+ output_context = nullcontext(cast(io.TextIOBase, sys.stdout))
+ else:
+ try:
+ output_context = output.open("w", encoding="utf-8")
+ except OSError as exc:
+ raise click.ClickException(f"failed to open output file '{output}': {exc.strerror or exc}") from exc
+
+ with output_context as output_stream:
+ LegacyModelTypeMigrationService(
+ engine=db.engine,
+ apply=apply,
+ concurrency=concurrency,
+ output=cast(io.TextIOBase, output_stream),
+ tables=normalized_tables or None,
+ model_types=selected_model_types,
+ tenant_ids=tenant_ids,
+ ).migrate()
+
+
+data_migrate.add_command(legacy_model_types)
+data_migrate.add_command(migrate_dataset_permissions_to_rbac)
+data_migrate.add_command(migrate_only_me_resource_whitelist_scopes_to_automatic_include)
+
+
def _scripted_export_template() -> dict[str, Any]:
return {
"source_tenant": {
diff --git a/api/tests/unit_tests/commands/test_data_migration_cli_registration.py b/api/tests/unit_tests/commands/test_data_migration_cli_registration.py
new file mode 100644
index 00000000000..af9028186a9
--- /dev/null
+++ b/api/tests/unit_tests/commands/test_data_migration_cli_registration.py
@@ -0,0 +1,94 @@
+"""Keep migration command ownership and the public Flask CLI layout stable."""
+
+import pytest
+
+import commands
+from dify_app import DifyApp
+from extensions import ext_commands
+
+
+def migration_app() -> DifyApp:
+ app = DifyApp(__name__)
+ ext_commands.init_app(app)
+ return app
+
+
+def test_migration_commands_are_registered_from_consolidated_module(migration_app: DifyApp) -> None:
+ expected_commands = {
+ "data-migrate": commands.data_migrate,
+ "app-migration-template": commands.export_migration_data_template,
+ "export-app-migration": commands.export_migration_data,
+ "import-app-migration": commands.import_migration_data,
+ "app-migration-wizard": commands.migration_data_wizard,
+ }
+
+ for name, command in expected_commands.items():
+ assert migration_app.cli.commands[name] is command
+ assert command.callback is not None
+ assert command.callback.__module__ == "commands.data_migration"
+
+ assert commands.legacy_model_types.callback is not None
+ assert commands.legacy_model_types.callback.__module__ == "commands.data_migration"
+ assert "legacy-model-types" not in migration_app.cli.commands
+
+
+def test_data_migrate_keeps_existing_subcommands_and_top_level_rbac_command(migration_app: DifyApp) -> None:
+ assert commands.data_migrate.commands == {
+ "legacy-model-types": commands.legacy_model_types,
+ "rbac-migrate-dataset-permissions": commands.migrate_dataset_permissions_to_rbac,
+ "rbac-migrate-resource-whitelist-scopes": (
+ commands.migrate_only_me_resource_whitelist_scopes_to_automatic_include
+ ),
+ }
+ assert (
+ migration_app.cli.commands["rbac-migrate-dataset-permissions"] is commands.migrate_dataset_permissions_to_rbac
+ )
+
+
+ ("command_path", "expected_help"),
+ [
+ (
+ ("data-migrate",),
+ ("legacy-model-types", "rbac-migrate-dataset-permissions", "rbac-migrate-resource-whitelist-scopes"),
+ ),
+ (
+ ("data-migrate", "legacy-model-types"),
+ ("--apply", "--tables", "--model-types", "--tenant-id-file", "--output", "--concurrency"),
+ ),
+ (
+ ("data-migrate", "rbac-migrate-dataset-permissions"),
+ ("--tenant-id", "--batch-size", "--dry-run"),
+ ),
+ (
+ ("data-migrate", "rbac-migrate-resource-whitelist-scopes"),
+ ("--tenant-id", "--resource-type", "--batch-size", "--dry-run"),
+ ),
+ (("app-migration-template",), ("--output", "--overwrite")),
+ (("export-app-migration",), ("--input", "--output", "--overwrite")),
+ (
+ ("import-app-migration",),
+ (
+ "--input",
+ "--target-tenant",
+ "--operator-email",
+ "--id-strategy",
+ "--conflict-strategy",
+ "--create-app-api-token-on-import",
+ "--no-create-app-api-token-on-import",
+ ),
+ ),
+ (("app-migration-wizard",), ("Interactively export workflow migration data.",)),
+ ],
+)
+def test_migration_command_help_remains_available(
+ migration_app: DifyApp,
+ command_path: tuple[str, ...],
+ expected_help: tuple[str, ...],
+) -> None:
+ result = migration_app.test_cli_runner().invoke(args=[*command_path, "--help"])
+
+ assert result.exit_code == 0, result.output
+ for text in expected_help:
+ assert text in result.output
diff --git a/api/tests/unit_tests/commands/test_legacy_model_type_migration.py b/api/tests/unit_tests/commands/test_legacy_model_type_migration.py
index e9dc551ad8a..6ebac69bcba 100644
--- a/api/tests/unit_tests/commands/test_legacy_model_type_migration.py
+++ b/api/tests/unit_tests/commands/test_legacy_model_type_migration.py
@@ -62,10 +62,10 @@ def dirty_fixture(sqlite_engine: sa.Engine) -> DirtyDataFixture:
@pytest.fixture
def command_module() -> ModuleType:
try:
- return importlib.import_module("commands.data_migrate")
+ return importlib.import_module("commands.data_migration")
except ModuleNotFoundError as exc: # pragma: no cover - explicit TDD failure path
pytest.fail(
- "commands.data_migrate is missing. "
+ "commands.data_migration is missing. "
"Implement the `flask data-migrate legacy-model-types` command group before running these tests."
)
@@ -823,7 +823,7 @@ def test_data_migrate_command_defaults_concurrency_from_cpu_count_or_falls_back_
expected_concurrency: int,
) -> None:
service_calls: list[dict[str, object]] = []
- command_module = importlib.import_module("commands.data_migrate")
+ command_module = importlib.import_module("commands.data_migration")
class FakeService:
def __init__(