Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(api): consolidate data migration CLI commands (#43483)
非法操作c60b3962df79
diff --git a/api/commands/__init__.py b/api/commands/__init__.pyindex 14bffa072e3..c5c9be55bcb 100644--- a/api/commands/__init__.py+++ b/api/commands/__init__.py@@ -3,11 +3,12 @@ CLI command modules extracted from `commands.py`."""from .account import create_tenant, reset_email, reset_password-from .data_migrate import data_migrate, legacy_model_typesfrom .data_migration import (+ data_migrate,export_migration_data,export_migration_data_template,import_migration_data,+ legacy_model_types,migration_data_wizard,)from .plugin import (diff --git a/api/commands/data_migrate.py b/api/commands/data_migrate.pydeleted file mode 100644index f1a7299b5c3..00000000000--- a/api/commands/data_migrate.py+++ /dev/null@@ -1,186 +0,0 @@-import io-import os-import sys-from contextlib import AbstractContextManager, nullcontext-from pathlib import Path-from typing import cast--import click--from commands.rbac import (- migrate_dataset_permissions_to_rbac,- migrate_only_me_resource_whitelist_scopes_to_automatic_include,- migrate_resource_whitelist_scopes_to_automatic_include, # noqa: F401-)-from extensions.ext_database import db-from graphon.model_runtime.entities.model_entities import ModelType-from services.legacy_model_type_migration import (- VALID_TABLE_NAMES,- LegacyModelTypeMigrationService,- load_tenant_ids_from_file,-)--_SUPPORTED_MODEL_TYPE_CHOICES = (- ModelType.LLM.value,- ModelType.TEXT_EMBEDDING.value,- ModelType.RERANK.value,-)-_DEFAULT_CONCURRENCY = os.cpu_count() or 1---def _normalize_multi_value_option(- values: tuple[str, ...],- *,- valid_values: tuple[str, ...],- option_name: str,-) -> tuple[str, ...]:- normalized_values: list[str] = []- seen_values: set[str] = set()-- for value in values:- for item in value.split(","):- normalized_item = item.strip()- if not normalized_item:- continue- if normalized_item not in valid_values:- raise click.BadParameter(- f"invalid value '{normalized_item}'. valid values: {', '.join(valid_values)}",- param_hint=option_name,- )- if normalized_item in seen_values:- continue- seen_values.add(normalized_item)- normalized_values.append(normalized_item)-- return tuple(normalized_values)--- "data-migrate",- help="Online data migration commands.",-)-def data_migrate() -> None:- """Namespace for production data migration commands."""--- "legacy-model-types",- help=(- "Migrate legacy provider model_type values to canonical values. "- "Default is dry-run and emits JSON lines only. "- "If --tables includes provider_model_credentials, the command may also update "- "provider_models and load_balancing_model_configs references so merged credentials stay reachable."- ),-)- "--apply",- is_flag=True,- default=False,- help="Apply the migration. Default is dry-run.",-)- "--tables",- "tables",- multiple=True,- type=str,- help=(- "Limit migration to specific tables. Accepts comma-separated values or repeated flags.\n"- "\n"- "Options: load_balancing_model_configs, provider_model_credentials, "- "provider_model_settings, provider_models, tenant_default_models.\n\n"- "When provider_model_credentials is selected, provider_models and "- "load_balancing_model_configs may also be updated for credential reference rewrites.\n"- "\n"- "If unspecified, all relevant tables are migrated."- ),-)- "--model-types",- "model_types",- multiple=True,- type=str,- help=(- "Canonical model types to migrate. Accepts comma-separated values or repeated flags.\n"- "\n"- "Options: llm,text-embedding,rerank\n"- "\n"- "If unspecified, all relevant legacy model types are migrated."- ),-)- "--tenant-id-file",- type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),- help="Optional file containing tenant ids, one per line.",-)- "--output",- type=click.Path(dir_okay=False, resolve_path=True, path_type=Path),- help=(- "Optional file path for JSON lines event logs. Defaults to stdout.\n"- "It's highly recommended to save the event logs to a file and preserve it for a period of time."- ),-)- "--concurrency",- type=click.IntRange(min=1),- default=_DEFAULT_CONCURRENCY,- show_default=True,- help="Number of tenant-level worker threads to run in parallel.",-)-def legacy_model_types(- apply: bool,- tables: tuple[str, ...],- model_types: tuple[str, ...],- tenant_id_file: str | None,- output: Path | None,- concurrency: int = _DEFAULT_CONCURRENCY,-) -> None:- """- Migrate legacy provider-related model_type values and emit JSON lines events.- """-- normalized_tables = _normalize_multi_value_option(- tables,- valid_values=VALID_TABLE_NAMES,- option_name="--tables",- )- normalized_model_types = _normalize_multi_value_option(- model_types,- valid_values=_SUPPORTED_MODEL_TYPE_CHOICES,- option_name="--model-types",- )- selected_model_types = (- tuple(ModelType(model_type) for model_type in normalized_model_types)- if normalized_model_types- else (- ModelType.LLM,- ModelType.TEXT_EMBEDDING,- ModelType.RERANK,- )- )- tenant_ids = load_tenant_ids_from_file(tenant_id_file) if tenant_id_file else None-- output_context: AbstractContextManager[io.TextIOBase]- if output is None:- output_context = nullcontext(cast(io.TextIOBase, sys.stdout))- else:- try:- output_context = output.open("w", encoding="utf-8")- except OSError as exc:- raise click.ClickException(f"failed to open output file '{output}': {exc.strerror or exc}") from exc-- with output_context as output_stream:- LegacyModelTypeMigrationService(- engine=db.engine,- apply=apply,- concurrency=concurrency,- output=cast(io.TextIOBase, output_stream),- tables=normalized_tables or None,- model_types=selected_model_types,- tenant_ids=tenant_ids,- ).migrate()---data_migrate.add_command(legacy_model_types)-data_migrate.add_command(migrate_dataset_permissions_to_rbac)-data_migrate.add_command(migrate_only_me_resource_whitelist_scopes_to_automatic_include)diff --git a/api/commands/data_migration.py b/api/commands/data_migration.pyindex 8c2627601a6..6bd36ee2200 100644--- a/api/commands/data_migration.py+++ b/api/commands/data_migration.py@@ -1,6 +1,12 @@+"""Online data migration and application import/export CLI commands."""+from __future__ import annotations+import ioimport json+import os+import sys+from contextlib import AbstractContextManager, nullcontextfrom datetime import datetimefrom pathlib import Pathfrom typing import Any, cast@@ -11,8 +17,14 @@ import sqlalchemy as saimport yamlfrom sqlalchemy.orm import Session+from commands.rbac import (+ migrate_dataset_permissions_to_rbac,+ migrate_only_me_resource_whitelist_scopes_to_automatic_include,+ migrate_resource_whitelist_scopes_to_automatic_include, # noqa: F401+)from core.db.session_factory import session_factoryfrom extensions.ext_database import db+from graphon.model_runtime.entities.model_entities import ModelTypefrom models import Tenantfrom models.model import Appfrom models.tools import ApiToolProvider, MCPToolProvider, WorkflowToolProvider@@ -29,7 +41,18 @@ from services.data_migration.export_service import ExportConfigParser, Migrationfrom services.data_migration.import_service import ImportRequest, MigrationImportServicefrom services.data_migration.package_service import MigrationPackageServicefrom services.data_migration.report_service import MigrationReportService+from services.legacy_model_type_migration import (+ VALID_TABLE_NAMES,+ LegacyModelTypeMigrationService,+ load_tenant_ids_from_file,+)+_SUPPORTED_MODEL_TYPE_CHOICES = (+ ModelType.LLM.value,+ ModelType.TEXT_EMBEDDING.value,+ ModelType.RERANK.value,+)+_DEFAULT_CONCURRENCY = os.cpu_count() or 1ID_STRATEGY_CHOICES = ["preserve-id", "generate-new-id"]CONFLICT_STRATEGY_CHOICES = ["fail", "skip", "update"]SUPPORTED_WIZARD_APP_MODES = ["workflow", "advanced-chat"]@@ -37,6 +60,164 @@ WizardToolMap = dict[str, dict[str, str | None]]WizardToolSelection = dict[str, list[str]]+def _normalize_multi_value_option(+ values: tuple[str, ...],+ *,+ valid_values: tuple[str, ...],+ option_name: str,+) -> tuple[str, ...]:+ normalized_values: list[str] = []+ seen_values: set[str] = set()++ for value in values:+ for item in value.split(","):+ normalized_item = item.strip()+ if not normalized_item:+ continue+ if normalized_item not in valid_values:+ raise click.BadParameter(+ f"invalid value '{normalized_item}'. valid values: {', '.join(valid_values)}",+ param_hint=option_name,+ )+ if normalized_item in seen_values:+ continue+ seen_values.add(normalized_item)+ normalized_values.append(normalized_item)++ return tuple(normalized_values)+++ "data-migrate",+ help="Online data migration commands.",+)+def data_migrate() -> None:+ """Namespace for production data migration commands."""+++ "legacy-model-types",+ help=(+ "Migrate legacy provider model_type values to canonical values. "+ "Default is dry-run and emits JSON lines only. "+ "If --tables includes provider_model_credentials, the command may also update "+ "provider_models and load_balancing_model_configs references so merged credentials stay reachable."+ ),+)+ "--apply",+ is_flag=True,+ default=False,+ help="Apply the migration. Default is dry-run.",+)+ "--tables",+ "tables",+ multiple=True,+ type=str,+ help=(+ "Limit migration to specific tables. Accepts comma-separated values or repeated flags.\n"+ "\n"+ "Options: load_balancing_model_configs, provider_model_credentials, "+ "provider_model_settings, provider_models, tenant_default_models.\n\n"+ "When provider_model_credentials is selected, provider_models and "+ "load_balancing_model_configs may also be updated for credential reference rewrites.\n"+ "\n"+ "If unspecified, all relevant tables are migrated."+ ),+)+ "--model-types",+ "model_types",+ multiple=True,+ type=str,+ help=(+ "Canonical model types to migrate. Accepts comma-separated values or repeated flags.\n"+ "\n"+ "Options: llm,text-embedding,rerank\n"+ "\n"+ "If unspecified, all relevant legacy model types are migrated."+ ),+)+ "--tenant-id-file",+ type=click.Path(exists=True, dir_okay=False, readable=True, resolve_path=True),+ help="Optional file containing tenant ids, one per line.",+)+ "--output",+ type=click.Path(dir_okay=False, resolve_path=True, path_type=Path),+ help=(+ "Optional file path for JSON lines event logs. Defaults to stdout.\n"+ "It's highly recommended to save the event logs to a file and preserve it for a period of time."+ ),+)+ "--concurrency",+ type=click.IntRange(min=1),+ default=_DEFAULT_CONCURRENCY,+ show_default=True,+ help="Number of tenant-level worker threads to run in parallel.",+)+def legacy_model_types(+ apply: bool,+ tables: tuple[str, ...],+ model_types: tuple[str, ...],+ tenant_id_file: str | None,+ output: Path | None,+ concurrency: int = _DEFAULT_CONCURRENCY,+) -> None:+ """+ Migrate legacy provider-related model_type values and emit JSON lines events.+ """++ normalized_tables = _normalize_multi_value_option(+ tables,+ valid_values=VALID_TABLE_NAMES,+ option_name="--tables",+ )+ normalized_model_types = _normalize_multi_value_option(+ model_types,+ valid_values=_SUPPORTED_MODEL_TYPE_CHOICES,+ option_name="--model-types",+ )+ selected_model_types = (+ tuple(ModelType(model_type) for model_type in normalized_model_types)+ if normalized_model_types+ else (+ ModelType.LLM,+ ModelType.TEXT_EMBEDDING,+ ModelType.RERANK,+ )+ )+ tenant_ids = load_tenant_ids_from_file(tenant_id_file) if tenant_id_file else None++ output_context: AbstractContextManager[io.TextIOBase]+ if output is None:+ output_context = nullcontext(cast(io.TextIOBase, sys.stdout))+ else:+ try:+ output_context = output.open("w", encoding="utf-8")+ except OSError as exc:+ raise click.ClickException(f"failed to open output file '{output}': {exc.strerror or exc}") from exc++ with output_context as output_stream:+ LegacyModelTypeMigrationService(+ engine=db.engine,+ apply=apply,+ concurrency=concurrency,+ output=cast(io.TextIOBase, output_stream),+ tables=normalized_tables or None,+ model_types=selected_model_types,+ tenant_ids=tenant_ids,+ ).migrate()+++data_migrate.add_command(legacy_model_types)+data_migrate.add_command(migrate_dataset_permissions_to_rbac)+data_migrate.add_command(migrate_only_me_resource_whitelist_scopes_to_automatic_include)++def _scripted_export_template() -> dict[str, Any]:return {"source_tenant": {diff --git a/api/tests/unit_tests/commands/test_data_migration_cli_registration.py b/api/tests/unit_tests/commands/test_data_migration_cli_registration.pynew file mode 100644index 00000000000..af9028186a9--- /dev/null+++ b/api/tests/unit_tests/commands/test_data_migration_cli_registration.py@@ -0,0 +1,94 @@+"""Keep migration command ownership and the public Flask CLI layout stable."""++import pytest++import commands+from dify_app import DifyApp+from extensions import ext_commands+++def migration_app() -> DifyApp:+ app = DifyApp(__name__)+ ext_commands.init_app(app)+ return app+++def test_migration_commands_are_registered_from_consolidated_module(migration_app: DifyApp) -> None:+ expected_commands = {+ "data-migrate": commands.data_migrate,+ "app-migration-template": commands.export_migration_data_template,+ "export-app-migration": commands.export_migration_data,+ "import-app-migration": commands.import_migration_data,+ "app-migration-wizard": commands.migration_data_wizard,+ }++ for name, command in expected_commands.items():+ assert migration_app.cli.commands[name] is command+ assert command.callback is not None+ assert command.callback.__module__ == "commands.data_migration"++ assert commands.legacy_model_types.callback is not None+ assert commands.legacy_model_types.callback.__module__ == "commands.data_migration"+ assert "legacy-model-types" not in migration_app.cli.commands+++def test_data_migrate_keeps_existing_subcommands_and_top_level_rbac_command(migration_app: DifyApp) -> None:+ assert commands.data_migrate.commands == {+ "legacy-model-types": commands.legacy_model_types,+ "rbac-migrate-dataset-permissions": commands.migrate_dataset_permissions_to_rbac,+ "rbac-migrate-resource-whitelist-scopes": (+ commands.migrate_only_me_resource_whitelist_scopes_to_automatic_include+ ),+ }+ assert (+ migration_app.cli.commands["rbac-migrate-dataset-permissions"] is commands.migrate_dataset_permissions_to_rbac+ )+++ ("command_path", "expected_help"),+ [+ (+ ("data-migrate",),+ ("legacy-model-types", "rbac-migrate-dataset-permissions", "rbac-migrate-resource-whitelist-scopes"),+ ),+ (+ ("data-migrate", "legacy-model-types"),+ ("--apply", "--tables", "--model-types", "--tenant-id-file", "--output", "--concurrency"),+ ),+ (+ ("data-migrate", "rbac-migrate-dataset-permissions"),+ ("--tenant-id", "--batch-size", "--dry-run"),+ ),+ (+ ("data-migrate", "rbac-migrate-resource-whitelist-scopes"),+ ("--tenant-id", "--resource-type", "--batch-size", "--dry-run"),+ ),+ (("app-migration-template",), ("--output", "--overwrite")),+ (("export-app-migration",), ("--input", "--output", "--overwrite")),+ (+ ("import-app-migration",),+ (+ "--input",+ "--target-tenant",+ "--operator-email",+ "--id-strategy",+ "--conflict-strategy",+ "--create-app-api-token-on-import",+ "--no-create-app-api-token-on-import",+ ),+ ),+ (("app-migration-wizard",), ("Interactively export workflow migration data.",)),+ ],+)+def test_migration_command_help_remains_available(+ migration_app: DifyApp,+ command_path: tuple[str, ...],+ expected_help: tuple[str, ...],+) -> None:+ result = migration_app.test_cli_runner().invoke(args=[*command_path, "--help"])++ assert result.exit_code == 0, result.output+ for text in expected_help:+ assert text in result.outputdiff --git a/api/tests/unit_tests/commands/test_legacy_model_type_migration.py b/api/tests/unit_tests/commands/test_legacy_model_type_migration.pyindex e9dc551ad8a..6ebac69bcba 100644--- a/api/tests/unit_tests/commands/test_legacy_model_type_migration.py+++ b/api/tests/unit_tests/commands/test_legacy_model_type_migration.py@@ -62,10 +62,10 @@ def dirty_fixture(sqlite_engine: sa.Engine) -> DirtyDataFixture:@pytest.fixturedef command_module() -> ModuleType:try:- return importlib.import_module("commands.data_migrate")+ return importlib.import_module("commands.data_migration")except ModuleNotFoundError as exc: # pragma: no cover - explicit TDD failure pathpytest.fail(- "commands.data_migrate is missing. "+ "commands.data_migration is missing. ""Implement the `flask data-migrate legacy-model-types` command group before running these tests.")@@ -823,7 +823,7 @@ def test_data_migrate_command_defaults_concurrency_from_cpu_count_or_falls_back_expected_concurrency: int,) -> None:service_calls: list[dict[str, object]] = []- command_module = importlib.import_module("commands.data_migrate")+ command_module = importlib.import_module("commands.data_migration")class FakeService:def __init__(