Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

feat: add miss index (#44982)

wangxiaolei4eef080524ba
diff --git a/api/migrations/versions/2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py b/api/migrations/versions/2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py
new file mode 100644
index 00000000000..0a7e620f454
--- /dev/null
+++ b/api/migrations/versions/2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py
@@ -0,0 +1,121 @@
+"""Add reference indexes used by application storage cleanup.
+
+Revision ID: d7e9a1c3f502
+Revises: b9d3f7a5c102
+Create Date: 2026-10-28 12:00:00
+
+Online upgrades validate and reuse existing indexes, including PostgreSQL validity.
+Offline SQL emits unconditional DDL for databases without these indexes. It fails
+on existing names instead of silently accepting an invalid or conflicting index;
+use online upgrades to reconcile indexes already created by SRE.
+Concurrent DDL commits independently of the revision stamp. Retrying preserves
+completed indexes. If a build leaves an invalid index, inspect its definition
+and ensure no build is still running, then DROP INDEX CONCURRENTLY that specific
+invalid index outside a transaction and retry. Never drop a healthy index as a
+routine recovery step. Resolve conflicting definitions manually before retrying.
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "d7e9a1c3f502"
+down_revision = "b9d3f7a5c102"
+branch_labels = None
+depends_on = None
+
+INDEXES = (
+ ("message_files", "message_file_upload_file_id_idx", "upload_file_id"),
+ ("workflow_node_execution_offload", "workflow_node_execution_offload_file_id_idx", "file_id"),
+ ("workflow_draft_variable_files", "workflow_draft_variable_files_upload_file_id_idx", "upload_file_id"),
+)
+
+# Look up the schema-wide relation name, not just indexes of the expected table:
+# a same-name index on another table must also be rejected. Check key options
+# explicitly so a different ordering, collation or operator class is not reused.
+_PG_INDEX = sa.text("""
+ SELECT i.indisvalid, i.indisready, i.indislive,
+ t.relname = :table_name AND t.relnamespace = c.relnamespace
+ AND c.relkind = 'i' AND am.amname = 'btree'
+ AND NOT i.indisunique AND NOT i.indisexclusion
+ AND i.indnatts = 1 AND i.indnkeyatts = 1
+ AND i.indexprs IS NULL AND i.indpred IS NULL
+ AND a.attname = :column_name AND i.indoption[0] = 0
+ AND opc.opcdefault AND i.indcollation[0] = a.attcollation AS matches
+ FROM pg_catalog.pg_class c
+ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+ LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid
+ LEFT JOIN pg_catalog.pg_class t ON t.oid = i.indrelid
+ LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam
+ LEFT JOIN pg_catalog.pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = i.indkey[0]
+ LEFT JOIN pg_catalog.pg_opclass opc ON opc.oid = i.indclass[0]
+ WHERE n.nspname = :schema_name AND c.relname = :index_name
+""")
+
+
+def _index_exists(table: str, name: str, column: str, schema: str) -> bool:
+ bind = op.get_bind()
+ if bind.dialect.name == "postgresql":
+ row = (
+ bind.execute(
+ _PG_INDEX,
+ {"schema_name": schema, "table_name": table, "index_name": name, "column_name": column},
+ )
+ .mappings()
+ .one_or_none()
+ )
+ if row is None:
+ return False
+ valid = all(row[key] for key in ("indisvalid", "indisready", "indislive", "matches"))
+ else:
+ indexes = sa.inspect(bind).get_indexes(table, schema=schema)
+ index = next((index for index in indexes if index["name"] == name), None)
+ if index is None:
+ return False
+ valid = index["column_names"] == [column] and not index["unique"] and not index.get("dialect_options")
+ if not valid:
+ raise RuntimeError(
+ f"Index {schema}.{name} is invalid or differs from the required non-unique btree "
+ f"index on {table}({column}). Inspect its definition and build status; resolve "
+ "the conflict before retrying. For an abandoned invalid PostgreSQL build, "
+ "DROP INDEX CONCURRENTLY outside a transaction, then retry this migration."
+ )
+ return True
+
+
+def _schema() -> str:
+ schema = sa.inspect(op.get_bind()).default_schema_name
+ if schema is None:
+ raise RuntimeError("Cannot determine the migration's default schema")
+ return schema
+
+
+def upgrade() -> None:
+ offline = op.get_context().as_sql
+ schema = None if offline else _schema()
+ # Preflight every name before starting any expensive, independently committed DDL.
+ missing = [
+ (table, name, column)
+ for table, name, column in INDEXES
+ if schema is None or not _index_exists(table, name, column, schema)
+ ]
+ for table, name, column in missing:
+ if op.get_bind().dialect.name == "postgresql":
+ with op.get_context().autocommit_block():
+ op.create_index(name, table, [column], schema=schema, postgresql_concurrently=True)
+ else:
+ op.create_index(name, table, [column], schema=schema)
+ if schema is not None and not _index_exists(table, name, column, schema):
+ raise RuntimeError(f"Index {schema}.{name} was not created")
+
+
+def downgrade() -> None:
+ schema = None if op.get_context().as_sql else _schema()
+ existing = [
+ (table, name) for table, name, column in INDEXES if schema is None or _index_exists(table, name, column, schema)
+ ]
+ for table, name in reversed(existing):
+ if op.get_bind().dialect.name == "postgresql":
+ with op.get_context().autocommit_block():
+ op.drop_index(name, table_name=table, schema=schema, postgresql_concurrently=True)
+ else:
+ op.drop_index(name, table_name=table, schema=schema)
diff --git a/api/models/model.py b/api/models/model.py
index f68d07ea050..c7637b034d2 100644
--- a/api/models/model.py
+++ b/api/models/model.py
@@ -1943,6 +1943,7 @@ class MessageFile(TypeBase):
__table_args__ = (
sa.PrimaryKeyConstraint("id", name="message_file_pkey"),
sa.Index("message_file_message_idx", "message_id"),
+ sa.Index("message_file_upload_file_id_idx", "upload_file_id"),
sa.Index("message_file_created_by_idx", "created_by"),
)
diff --git a/api/models/workflow.py b/api/models/workflow.py
index 3653a3f30ab..b3945a06794 100644
--- a/api/models/workflow.py
+++ b/api/models/workflow.py
@@ -1243,6 +1243,7 @@ class WorkflowNodeExecutionModel(TypeBase): # This model is expected to have `o
class WorkflowNodeExecutionOffload(TypeBase):
__tablename__ = "workflow_node_execution_offload"
__table_args__ = (
+ sa.Index("workflow_node_execution_offload_file_id_idx", "file_id"),
# PostgreSQL 14 treats NULL values as distinct in unique constraints by default,
# allowing multiple records with NULL values for the same column combination.
#
@@ -2073,6 +2074,7 @@ class WorkflowDraftVariableFile(TypeBase):
"""
__tablename__ = "workflow_draft_variable_files"
+ __table_args__ = (sa.Index("workflow_draft_variable_files_upload_file_id_idx", "upload_file_id"),)
# Primary key
id: Mapped[str] = mapped_column(StringUUID, primary_key=True, default_factory=lambda: str(uuidv7()), init=False)
diff --git a/api/tests/unit_tests/migrations/versions/test_2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py b/api/tests/unit_tests/migrations/versions/test_2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py
new file mode 100644
index 00000000000..34b8bc9462a
--- /dev/null
+++ b/api/tests/unit_tests/migrations/versions/test_2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py
@@ -0,0 +1,167 @@
+import importlib.util
+import shutil
+from io import StringIO
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+import sqlalchemy as sa
+from alembic import command
+from alembic.config import Config
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from flask import Flask
+from flask_migrate import Migrate
+from flask_sqlalchemy import SQLAlchemy
+from sqlalchemy import event
+
+from models.model import MessageFile
+from models.workflow import WorkflowDraftVariableFile, WorkflowNodeExecutionOffload
+
+
+def load_migration() -> ModuleType:
+ path = Path(__file__).resolve().parents[4] / (
+ "migrations/versions/2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py"
+ )
+ spec = importlib.util.spec_from_file_location("app_cleanup_indexes", path)
+ assert spec is not None
+ assert spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
[email protected]("existing_count", [0, 1, 3])
[email protected]("populated", [False, True])
+def test_upgrade_preserves_rows_and_reuses_indexes(existing_count: int, populated: bool) -> None:
+ migration = load_migration()
+ engine = sa.create_engine("sqlite://")
+ metadata = sa.MetaData()
+ for position, (table_name, name, column) in enumerate(migration.INDEXES):
+ table = sa.Table(table_name, metadata, sa.Column(column, sa.String), sa.Column("other", sa.String))
+ if position < existing_count:
+ sa.Index(name, table.c[column])
+ metadata.create_all(engine)
+ ddl: list[str] = []
+
+ @event.listens_for(engine, "before_cursor_execute")
+ def record_ddl(_conn, _cursor, statement: str, _parameters, _context, _executemany) -> None:
+ if statement.strip().startswith(("CREATE INDEX", "DROP INDEX")):
+ ddl.append(statement)
+
+ with engine.begin() as connection:
+ if populated:
+ for table in metadata.tables.values():
+ connection.execute(table.insert().values({next(iter(table.c)).name: "historical-file"}))
+ migration.__dict__["op"] = Operations(MigrationContext.configure(connection))
+ migration.upgrade()
+ migration.upgrade()
+ assert len(ddl) == 3 - existing_count
+ for table, name, column in migration.INDEXES:
+ indexes = sa.inspect(connection).get_indexes(table)
+ assert [(index["name"], index["column_names"]) for index in indexes] == [(name, [column])]
+ assert connection.scalar(sa.select(sa.func.count()).select_from(metadata.tables[table])) == int(populated)
+ migration.downgrade()
+ migration.downgrade()
+ assert len(ddl) == 6 - existing_count
+ for table, _, _ in migration.INDEXES:
+ assert sa.inspect(connection).get_indexes(table) == []
+ engine.dispose()
+
+
[email protected]("kind", ["wrong_column", "unique", "partial"])
+def test_conflicting_index_fails_before_any_index_is_created(kind: str) -> None:
+ migration = load_migration()
+ engine = sa.create_engine("sqlite://")
+ metadata = sa.MetaData()
+ for table_name, _, column in migration.INDEXES:
+ sa.Table(table_name, metadata, sa.Column(column, sa.String), sa.Column("other", sa.String))
+ table_name, name, column = migration.INDEXES[-1]
+ table = metadata.tables[table_name]
+ sa.Index(
+ name,
+ table.c.other if kind == "wrong_column" else table.c[column],
+ unique=kind == "unique",
+ **({"sqlite_where": table.c[column].is_not(None)} if kind == "partial" else {}),
+ )
+ metadata.create_all(engine)
+ with engine.begin() as connection:
+ migration.__dict__["op"] = Operations(MigrationContext.configure(connection))
+ with pytest.raises(RuntimeError, match="invalid or differs"):
+ migration.upgrade()
+ assert sa.inspect(connection).get_indexes(migration.INDEXES[0][0]) == []
+ engine.dispose()
+
+
+def test_model_indexes_match_migration() -> None:
+ models = {
+ model.__tablename__: model for model in (MessageFile, WorkflowNodeExecutionOffload, WorkflowDraftVariableFile)
+ }
+ for table, name, column in load_migration().INDEXES:
+ model_table = models[table].__table__
+ assert isinstance(model_table, sa.Table)
+ index = next(index for index in model_table.indexes if index.name == name)
+ assert [col.name for col in index.columns] == [column]
+ assert not index.unique
+
+
[email protected]("dialect", ["postgresql", "mysql"])
+def test_offline_migration_emits_ddl_without_silently_skipping_existing_indexes(dialect: str) -> None:
+ migration = load_migration()
+ output = StringIO()
+ context = MigrationContext.configure(dialect_name=dialect, opts={"as_sql": True, "output_buffer": output})
+ migration.__dict__["op"] = Operations(context)
+ with context.begin_transaction():
+ migration.upgrade()
+ migration.downgrade()
+ sql = output.getvalue()
+ statements = [statement.strip() for statement in sql.split(";") if statement.strip()]
+ assert "IF NOT EXISTS" not in sql
+ assert sql.count("CREATE INDEX") == 3
+ assert sql.count("DROP INDEX") == 3
+ for table, name, column in migration.INDEXES:
+ if dialect == "postgresql":
+ create = f"CREATE INDEX CONCURRENTLY {name} ON {table} ({column})"
+ drop = f"DROP INDEX CONCURRENTLY {name}"
+ for ddl in (create, drop):
+ position = statements.index(ddl)
+ assert statements[position - 1] == "COMMIT"
+ assert statements[position + 1] == "BEGIN"
+ else:
+ assert f"CREATE INDEX {name} ON {table} ({column})" in statements
+ assert f"DROP INDEX {name} ON {table}" in statements
+ assert "CONCURRENTLY" not in sql
+
+
[email protected]("database_uri", ["postgresql://", "mysql+pymysql://"])
+def test_alembic_offline_upgrade_and_downgrade_stamp_revision(
+ database_uri: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ migrations = Path(__file__).resolve().parents[4] / "migrations"
+ # Exercise the real env.py and this revision without loading unrelated
+ # local migration branches. The parent is only a starting revision marker.
+ versions = tmp_path / "versions"
+ versions.mkdir()
+ shutil.copyfile(migrations / "env.py", tmp_path / "env.py")
+ filename = "2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py"
+ shutil.copyfile(migrations / "versions" / filename, versions / filename)
+ (versions / "parent.py").write_text('revision = "b9d3f7a5c102"\ndown_revision = None\n')
+ output = StringIO()
+ config = Config(str(migrations / "alembic.ini"), output_buffer=output)
+ config.set_main_option("script_location", str(tmp_path))
+ app = Flask(__name__)
+ app.config["SQLALCHEMY_DATABASE_URI"] = database_uri
+ database = SQLAlchemy(app)
+ Migrate(app, database)
+ monkeypatch.setattr("logging.config.fileConfig", lambda _filename: None)
+ with app.app_context():
+ command.upgrade(config, "b9d3f7a5c102:d7e9a1c3f502", sql=True)
+ command.downgrade(config, "d7e9a1c3f502:b9d3f7a5c102", sql=True)
+ database.engine.dispose()
+ sql = output.getvalue()
+ assert sql.count("CREATE INDEX") == 3
+ assert sql.count("DROP INDEX") == 3
+ upgrade_stamp = sql.index("UPDATE alembic_version SET version_num='d7e9a1c3f502'")
+ downgrade_stamp = sql.index("UPDATE alembic_version SET version_num='b9d3f7a5c102'")
+ assert sql.rindex("CREATE INDEX") < upgrade_stamp < sql.index("DROP INDEX")
+ assert sql.rindex("DROP INDEX") < downgrade_stamp
diff --git a/api/tests/unit_tests/services/test_app_storage_cleanup_service.py b/api/tests/unit_tests/services/test_app_storage_cleanup_service.py
index c8c03f835bb..89f69a93331 100644
--- a/api/tests/unit_tests/services/test_app_storage_cleanup_service.py
+++ b/api/tests/unit_tests/services/test_app_storage_cleanup_service.py
@@ -15,6 +15,7 @@ from extensions.storage.storage_type import StorageType
from graphon.entities.pause_reason import PauseReasonType
from graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatus
from graphon.file import FileTransferMethod, FileType
+from graphon.variables.types import SegmentType
from models.enums import (
ConversationFromSource,
CreatorUserRole,
@@ -25,6 +26,7 @@ from models.enums import (
from models.model import AppMode, Conversation, EndUser, Message, MessageFile, UploadFile
from models.tools import ToolFile
from models.workflow import (
+ WorkflowDraftVariableFile,
WorkflowNodeExecutionModel,
WorkflowNodeExecutionOffload,
WorkflowNodeExecutionTriggeredFrom,
@@ -557,3 +559,99 @@ def test_tool_with_conflicting_conversation_owner_is_retained(sqlite_session: Se
cleanup.cleanup_app_storage("tenant-1", "app-1")
storage.delete.assert_not_called()
assert sqlite_session.get(ToolFile, tool.id) is not None
+
+
[email protected]("message_owner", ["app-1", "app-2", None])
+def test_message_reference_protection_for_tools_and_uploads(
+ sqlite_session: Session, deleted_keys: list[str], message_owner: str | None
+) -> None:
+ user = _end_user(sqlite_session)
+ tool = ToolFile(
+ tenant_id="tenant-1",
+ user_id=user.id,
+ conversation_id=None,
+ file_key="tools/reference.json",
+ mimetype="application/json",
+ )
+ sqlite_session.add(tool)
+ file_id, offload_id = _offload(sqlite_session, "app-2", "uploads/reference.json")
+ offload = sqlite_session.get(WorkflowNodeExecutionOffload, offload_id)
+ assert offload is not None
+ sqlite_session.delete(offload)
+ upload = sqlite_session.get(UploadFile, file_id)
+ assert upload is not None
+ upload.created_by_role = CreatorUserRole.END_USER
+ upload.created_by = user.id
+ message_id = "orphan-message"
+ if message_owner is not None:
+ message = Message(
+ app_id=message_owner,
+ conversation_id="conversation",
+ _inputs={},
+ query="q",
+ message={},
+ message_unit_price=Decimal(0),
+ answer="a",
+ answer_unit_price=Decimal(0),
+ currency="USD",
+ from_source=ConversationFromSource.API,
+ )
+ sqlite_session.add(message)
+ sqlite_session.flush()
+ message_id = message.id
+ tool_id = tool.id
+ for record_id, method in ((tool_id, FileTransferMethod.TOOL_FILE), (file_id, FileTransferMethod.LOCAL_FILE)):
+ sqlite_session.add(
+ MessageFile(
+ message_id=message_id,
+ type=FileType.DOCUMENT,
+ transfer_method=method,
+ upload_file_id=record_id,
+ created_by_role=CreatorUserRole.END_USER,
+ created_by=user.id,
+ )
+ )
+ sqlite_session.commit()
+ for _ in range(2):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ retained = message_owner != "app-1"
+ sqlite_session.expire_all()
+ assert (sqlite_session.get(ToolFile, tool_id) is not None) == retained
+ assert (sqlite_session.get(UploadFile, file_id) is not None) == retained
+ assert sorted(deleted_keys) == ([] if retained else ["tools/reference.json", "uploads/reference.json"])
+
+
[email protected]("reference", ["offload", "draft", "foreign_tenant"])
+def test_end_user_upload_reference_and_tenant_protection(
+ sqlite_session: Session, deleted_keys: list[str], reference: str
+) -> None:
+ user = _end_user(sqlite_session)
+ file_id, offload_id = _offload(sqlite_session, "app-2", "uploads/protected.json")
+ upload = sqlite_session.get(UploadFile, file_id)
+ assert upload is not None
+ upload.created_by_role = CreatorUserRole.END_USER
+ upload.created_by = user.id
+ if reference != "offload":
+ offload = sqlite_session.get(WorkflowNodeExecutionOffload, offload_id)
+ assert offload is not None
+ sqlite_session.delete(offload)
+ if reference == "draft":
+ sqlite_session.add(
+ WorkflowDraftVariableFile(
+ tenant_id="tenant-1",
+ app_id="app-2",
+ user_id="account-1",
+ upload_file_id=file_id,
+ size=1,
+ length=None,
+ value_type=SegmentType.STRING,
+ )
+ )
+ elif reference == "foreign_tenant":
+ upload.tenant_id = "tenant-2"
+ sqlite_session.commit()
+ for _ in range(2):
+ cleanup.cleanup_app_storage("tenant-1", "app-1")
+ sqlite_session.expire_all()
+ assert sqlite_session.get(UploadFile, file_id) is not None
+ assert deleted_keys == []