Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
feat: add miss index (#44982)
wangxiaolei4eef080524ba
diff --git a/api/migrations/versions/2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py b/api/migrations/versions/2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.pynew file mode 100644index 00000000000..0a7e620f454--- /dev/null+++ b/api/migrations/versions/2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py@@ -0,0 +1,121 @@+"""Add reference indexes used by application storage cleanup.++Revision ID: d7e9a1c3f502+Revises: b9d3f7a5c102+Create Date: 2026-10-28 12:00:00++Online upgrades validate and reuse existing indexes, including PostgreSQL validity.+Offline SQL emits unconditional DDL for databases without these indexes. It fails+on existing names instead of silently accepting an invalid or conflicting index;+use online upgrades to reconcile indexes already created by SRE.+Concurrent DDL commits independently of the revision stamp. Retrying preserves+completed indexes. If a build leaves an invalid index, inspect its definition+and ensure no build is still running, then DROP INDEX CONCURRENTLY that specific+invalid index outside a transaction and retry. Never drop a healthy index as a+routine recovery step. Resolve conflicting definitions manually before retrying.+"""++import sqlalchemy as sa+from alembic import op++revision = "d7e9a1c3f502"+down_revision = "b9d3f7a5c102"+branch_labels = None+depends_on = None++INDEXES = (+ ("message_files", "message_file_upload_file_id_idx", "upload_file_id"),+ ("workflow_node_execution_offload", "workflow_node_execution_offload_file_id_idx", "file_id"),+ ("workflow_draft_variable_files", "workflow_draft_variable_files_upload_file_id_idx", "upload_file_id"),+)++# Look up the schema-wide relation name, not just indexes of the expected table:+# a same-name index on another table must also be rejected. Check key options+# explicitly so a different ordering, collation or operator class is not reused.+_PG_INDEX = sa.text("""+ SELECT i.indisvalid, i.indisready, i.indislive,+ t.relname = :table_name AND t.relnamespace = c.relnamespace+ AND c.relkind = 'i' AND am.amname = 'btree'+ AND NOT i.indisunique AND NOT i.indisexclusion+ AND i.indnatts = 1 AND i.indnkeyatts = 1+ AND i.indexprs IS NULL AND i.indpred IS NULL+ AND a.attname = :column_name AND i.indoption[0] = 0+ AND opc.opcdefault AND i.indcollation[0] = a.attcollation AS matches+ FROM pg_catalog.pg_class c+ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace+ LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid+ LEFT JOIN pg_catalog.pg_class t ON t.oid = i.indrelid+ LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam+ LEFT JOIN pg_catalog.pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = i.indkey[0]+ LEFT JOIN pg_catalog.pg_opclass opc ON opc.oid = i.indclass[0]+ WHERE n.nspname = :schema_name AND c.relname = :index_name+""")+++def _index_exists(table: str, name: str, column: str, schema: str) -> bool:+ bind = op.get_bind()+ if bind.dialect.name == "postgresql":+ row = (+ bind.execute(+ _PG_INDEX,+ {"schema_name": schema, "table_name": table, "index_name": name, "column_name": column},+ )+ .mappings()+ .one_or_none()+ )+ if row is None:+ return False+ valid = all(row[key] for key in ("indisvalid", "indisready", "indislive", "matches"))+ else:+ indexes = sa.inspect(bind).get_indexes(table, schema=schema)+ index = next((index for index in indexes if index["name"] == name), None)+ if index is None:+ return False+ valid = index["column_names"] == [column] and not index["unique"] and not index.get("dialect_options")+ if not valid:+ raise RuntimeError(+ f"Index {schema}.{name} is invalid or differs from the required non-unique btree "+ f"index on {table}({column}). Inspect its definition and build status; resolve "+ "the conflict before retrying. For an abandoned invalid PostgreSQL build, "+ "DROP INDEX CONCURRENTLY outside a transaction, then retry this migration."+ )+ return True+++def _schema() -> str:+ schema = sa.inspect(op.get_bind()).default_schema_name+ if schema is None:+ raise RuntimeError("Cannot determine the migration's default schema")+ return schema+++def upgrade() -> None:+ offline = op.get_context().as_sql+ schema = None if offline else _schema()+ # Preflight every name before starting any expensive, independently committed DDL.+ missing = [+ (table, name, column)+ for table, name, column in INDEXES+ if schema is None or not _index_exists(table, name, column, schema)+ ]+ for table, name, column in missing:+ if op.get_bind().dialect.name == "postgresql":+ with op.get_context().autocommit_block():+ op.create_index(name, table, [column], schema=schema, postgresql_concurrently=True)+ else:+ op.create_index(name, table, [column], schema=schema)+ if schema is not None and not _index_exists(table, name, column, schema):+ raise RuntimeError(f"Index {schema}.{name} was not created")+++def downgrade() -> None:+ schema = None if op.get_context().as_sql else _schema()+ existing = [+ (table, name) for table, name, column in INDEXES if schema is None or _index_exists(table, name, column, schema)+ ]+ for table, name in reversed(existing):+ if op.get_bind().dialect.name == "postgresql":+ with op.get_context().autocommit_block():+ op.drop_index(name, table_name=table, schema=schema, postgresql_concurrently=True)+ else:+ op.drop_index(name, table_name=table, schema=schema)diff --git a/api/models/model.py b/api/models/model.pyindex f68d07ea050..c7637b034d2 100644--- a/api/models/model.py+++ b/api/models/model.py@@ -1943,6 +1943,7 @@ class MessageFile(TypeBase):__table_args__ = (sa.PrimaryKeyConstraint("id", name="message_file_pkey"),sa.Index("message_file_message_idx", "message_id"),+ sa.Index("message_file_upload_file_id_idx", "upload_file_id"),sa.Index("message_file_created_by_idx", "created_by"),)diff --git a/api/models/workflow.py b/api/models/workflow.pyindex 3653a3f30ab..b3945a06794 100644--- a/api/models/workflow.py+++ b/api/models/workflow.py@@ -1243,6 +1243,7 @@ class WorkflowNodeExecutionModel(TypeBase): # This model is expected to have `oclass WorkflowNodeExecutionOffload(TypeBase):__tablename__ = "workflow_node_execution_offload"__table_args__ = (+ sa.Index("workflow_node_execution_offload_file_id_idx", "file_id"),# PostgreSQL 14 treats NULL values as distinct in unique constraints by default,# allowing multiple records with NULL values for the same column combination.#@@ -2073,6 +2074,7 @@ class WorkflowDraftVariableFile(TypeBase):"""__tablename__ = "workflow_draft_variable_files"+ __table_args__ = (sa.Index("workflow_draft_variable_files_upload_file_id_idx", "upload_file_id"),)# Primary keyid: Mapped[str] = mapped_column(StringUUID, primary_key=True, default_factory=lambda: str(uuidv7()), init=False)diff --git a/api/tests/unit_tests/migrations/versions/test_2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py b/api/tests/unit_tests/migrations/versions/test_2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.pynew file mode 100644index 00000000000..34b8bc9462a--- /dev/null+++ b/api/tests/unit_tests/migrations/versions/test_2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py@@ -0,0 +1,167 @@+import importlib.util+import shutil+from io import StringIO+from pathlib import Path+from types import ModuleType++import pytest+import sqlalchemy as sa+from alembic import command+from alembic.config import Config+from alembic.migration import MigrationContext+from alembic.operations import Operations+from flask import Flask+from flask_migrate import Migrate+from flask_sqlalchemy import SQLAlchemy+from sqlalchemy import event++from models.model import MessageFile+from models.workflow import WorkflowDraftVariableFile, WorkflowNodeExecutionOffload+++def load_migration() -> ModuleType:+ path = Path(__file__).resolve().parents[4] / (+ "migrations/versions/2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py"+ )+ spec = importlib.util.spec_from_file_location("app_cleanup_indexes", path)+ assert spec is not None+ assert spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module++[email protected]("existing_count", [0, 1, 3])[email protected]("populated", [False, True])+def test_upgrade_preserves_rows_and_reuses_indexes(existing_count: int, populated: bool) -> None:+ migration = load_migration()+ engine = sa.create_engine("sqlite://")+ metadata = sa.MetaData()+ for position, (table_name, name, column) in enumerate(migration.INDEXES):+ table = sa.Table(table_name, metadata, sa.Column(column, sa.String), sa.Column("other", sa.String))+ if position < existing_count:+ sa.Index(name, table.c[column])+ metadata.create_all(engine)+ ddl: list[str] = []++ @event.listens_for(engine, "before_cursor_execute")+ def record_ddl(_conn, _cursor, statement: str, _parameters, _context, _executemany) -> None:+ if statement.strip().startswith(("CREATE INDEX", "DROP INDEX")):+ ddl.append(statement)++ with engine.begin() as connection:+ if populated:+ for table in metadata.tables.values():+ connection.execute(table.insert().values({next(iter(table.c)).name: "historical-file"}))+ migration.__dict__["op"] = Operations(MigrationContext.configure(connection))+ migration.upgrade()+ migration.upgrade()+ assert len(ddl) == 3 - existing_count+ for table, name, column in migration.INDEXES:+ indexes = sa.inspect(connection).get_indexes(table)+ assert [(index["name"], index["column_names"]) for index in indexes] == [(name, [column])]+ assert connection.scalar(sa.select(sa.func.count()).select_from(metadata.tables[table])) == int(populated)+ migration.downgrade()+ migration.downgrade()+ assert len(ddl) == 6 - existing_count+ for table, _, _ in migration.INDEXES:+ assert sa.inspect(connection).get_indexes(table) == []+ engine.dispose()++[email protected]("kind", ["wrong_column", "unique", "partial"])+def test_conflicting_index_fails_before_any_index_is_created(kind: str) -> None:+ migration = load_migration()+ engine = sa.create_engine("sqlite://")+ metadata = sa.MetaData()+ for table_name, _, column in migration.INDEXES:+ sa.Table(table_name, metadata, sa.Column(column, sa.String), sa.Column("other", sa.String))+ table_name, name, column = migration.INDEXES[-1]+ table = metadata.tables[table_name]+ sa.Index(+ name,+ table.c.other if kind == "wrong_column" else table.c[column],+ unique=kind == "unique",+ **({"sqlite_where": table.c[column].is_not(None)} if kind == "partial" else {}),+ )+ metadata.create_all(engine)+ with engine.begin() as connection:+ migration.__dict__["op"] = Operations(MigrationContext.configure(connection))+ with pytest.raises(RuntimeError, match="invalid or differs"):+ migration.upgrade()+ assert sa.inspect(connection).get_indexes(migration.INDEXES[0][0]) == []+ engine.dispose()+++def test_model_indexes_match_migration() -> None:+ models = {+ model.__tablename__: model for model in (MessageFile, WorkflowNodeExecutionOffload, WorkflowDraftVariableFile)+ }+ for table, name, column in load_migration().INDEXES:+ model_table = models[table].__table__+ assert isinstance(model_table, sa.Table)+ index = next(index for index in model_table.indexes if index.name == name)+ assert [col.name for col in index.columns] == [column]+ assert not index.unique++[email protected]("dialect", ["postgresql", "mysql"])+def test_offline_migration_emits_ddl_without_silently_skipping_existing_indexes(dialect: str) -> None:+ migration = load_migration()+ output = StringIO()+ context = MigrationContext.configure(dialect_name=dialect, opts={"as_sql": True, "output_buffer": output})+ migration.__dict__["op"] = Operations(context)+ with context.begin_transaction():+ migration.upgrade()+ migration.downgrade()+ sql = output.getvalue()+ statements = [statement.strip() for statement in sql.split(";") if statement.strip()]+ assert "IF NOT EXISTS" not in sql+ assert sql.count("CREATE INDEX") == 3+ assert sql.count("DROP INDEX") == 3+ for table, name, column in migration.INDEXES:+ if dialect == "postgresql":+ create = f"CREATE INDEX CONCURRENTLY {name} ON {table} ({column})"+ drop = f"DROP INDEX CONCURRENTLY {name}"+ for ddl in (create, drop):+ position = statements.index(ddl)+ assert statements[position - 1] == "COMMIT"+ assert statements[position + 1] == "BEGIN"+ else:+ assert f"CREATE INDEX {name} ON {table} ({column})" in statements+ assert f"DROP INDEX {name} ON {table}" in statements+ assert "CONCURRENTLY" not in sql++[email protected]("database_uri", ["postgresql://", "mysql+pymysql://"])+def test_alembic_offline_upgrade_and_downgrade_stamp_revision(+ database_uri: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ migrations = Path(__file__).resolve().parents[4] / "migrations"+ # Exercise the real env.py and this revision without loading unrelated+ # local migration branches. The parent is only a starting revision marker.+ versions = tmp_path / "versions"+ versions.mkdir()+ shutil.copyfile(migrations / "env.py", tmp_path / "env.py")+ filename = "2026_10_28_1200-d7e9a1c3f502_add_app_cleanup_reference_indexes.py"+ shutil.copyfile(migrations / "versions" / filename, versions / filename)+ (versions / "parent.py").write_text('revision = "b9d3f7a5c102"\ndown_revision = None\n')+ output = StringIO()+ config = Config(str(migrations / "alembic.ini"), output_buffer=output)+ config.set_main_option("script_location", str(tmp_path))+ app = Flask(__name__)+ app.config["SQLALCHEMY_DATABASE_URI"] = database_uri+ database = SQLAlchemy(app)+ Migrate(app, database)+ monkeypatch.setattr("logging.config.fileConfig", lambda _filename: None)+ with app.app_context():+ command.upgrade(config, "b9d3f7a5c102:d7e9a1c3f502", sql=True)+ command.downgrade(config, "d7e9a1c3f502:b9d3f7a5c102", sql=True)+ database.engine.dispose()+ sql = output.getvalue()+ assert sql.count("CREATE INDEX") == 3+ assert sql.count("DROP INDEX") == 3+ upgrade_stamp = sql.index("UPDATE alembic_version SET version_num='d7e9a1c3f502'")+ downgrade_stamp = sql.index("UPDATE alembic_version SET version_num='b9d3f7a5c102'")+ assert sql.rindex("CREATE INDEX") < upgrade_stamp < sql.index("DROP INDEX")+ assert sql.rindex("DROP INDEX") < downgrade_stampdiff --git a/api/tests/unit_tests/services/test_app_storage_cleanup_service.py b/api/tests/unit_tests/services/test_app_storage_cleanup_service.pyindex c8c03f835bb..89f69a93331 100644--- a/api/tests/unit_tests/services/test_app_storage_cleanup_service.py+++ b/api/tests/unit_tests/services/test_app_storage_cleanup_service.py@@ -15,6 +15,7 @@ from extensions.storage.storage_type import StorageTypefrom graphon.entities.pause_reason import PauseReasonTypefrom graphon.enums import WorkflowExecutionStatus, WorkflowNodeExecutionStatusfrom graphon.file import FileTransferMethod, FileType+from graphon.variables.types import SegmentTypefrom models.enums import (ConversationFromSource,CreatorUserRole,@@ -25,6 +26,7 @@ from models.enums import (from models.model import AppMode, Conversation, EndUser, Message, MessageFile, UploadFilefrom models.tools import ToolFilefrom models.workflow import (+ WorkflowDraftVariableFile,WorkflowNodeExecutionModel,WorkflowNodeExecutionOffload,WorkflowNodeExecutionTriggeredFrom,@@ -557,3 +559,99 @@ def test_tool_with_conflicting_conversation_owner_is_retained(sqlite_session: Secleanup.cleanup_app_storage("tenant-1", "app-1")storage.delete.assert_not_called()assert sqlite_session.get(ToolFile, tool.id) is not None++[email protected]("message_owner", ["app-1", "app-2", None])+def test_message_reference_protection_for_tools_and_uploads(+ sqlite_session: Session, deleted_keys: list[str], message_owner: str | None+) -> None:+ user = _end_user(sqlite_session)+ tool = ToolFile(+ tenant_id="tenant-1",+ user_id=user.id,+ conversation_id=None,+ file_key="tools/reference.json",+ mimetype="application/json",+ )+ sqlite_session.add(tool)+ file_id, offload_id = _offload(sqlite_session, "app-2", "uploads/reference.json")+ offload = sqlite_session.get(WorkflowNodeExecutionOffload, offload_id)+ assert offload is not None+ sqlite_session.delete(offload)+ upload = sqlite_session.get(UploadFile, file_id)+ assert upload is not None+ upload.created_by_role = CreatorUserRole.END_USER+ upload.created_by = user.id+ message_id = "orphan-message"+ if message_owner is not None:+ message = Message(+ app_id=message_owner,+ conversation_id="conversation",+ _inputs={},+ query="q",+ message={},+ message_unit_price=Decimal(0),+ answer="a",+ answer_unit_price=Decimal(0),+ currency="USD",+ from_source=ConversationFromSource.API,+ )+ sqlite_session.add(message)+ sqlite_session.flush()+ message_id = message.id+ tool_id = tool.id+ for record_id, method in ((tool_id, FileTransferMethod.TOOL_FILE), (file_id, FileTransferMethod.LOCAL_FILE)):+ sqlite_session.add(+ MessageFile(+ message_id=message_id,+ type=FileType.DOCUMENT,+ transfer_method=method,+ upload_file_id=record_id,+ created_by_role=CreatorUserRole.END_USER,+ created_by=user.id,+ )+ )+ sqlite_session.commit()+ for _ in range(2):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ retained = message_owner != "app-1"+ sqlite_session.expire_all()+ assert (sqlite_session.get(ToolFile, tool_id) is not None) == retained+ assert (sqlite_session.get(UploadFile, file_id) is not None) == retained+ assert sorted(deleted_keys) == ([] if retained else ["tools/reference.json", "uploads/reference.json"])++[email protected]("reference", ["offload", "draft", "foreign_tenant"])+def test_end_user_upload_reference_and_tenant_protection(+ sqlite_session: Session, deleted_keys: list[str], reference: str+) -> None:+ user = _end_user(sqlite_session)+ file_id, offload_id = _offload(sqlite_session, "app-2", "uploads/protected.json")+ upload = sqlite_session.get(UploadFile, file_id)+ assert upload is not None+ upload.created_by_role = CreatorUserRole.END_USER+ upload.created_by = user.id+ if reference != "offload":+ offload = sqlite_session.get(WorkflowNodeExecutionOffload, offload_id)+ assert offload is not None+ sqlite_session.delete(offload)+ if reference == "draft":+ sqlite_session.add(+ WorkflowDraftVariableFile(+ tenant_id="tenant-1",+ app_id="app-2",+ user_id="account-1",+ upload_file_id=file_id,+ size=1,+ length=None,+ value_type=SegmentType.STRING,+ )+ )+ elif reference == "foreign_tenant":+ upload.tenant_id = "tenant-2"+ sqlite_session.commit()+ for _ in range(2):+ cleanup.cleanup_app_storage("tenant-1", "app-1")+ sqlite_session.expire_all()+ assert sqlite_session.get(UploadFile, file_id) is not None+ assert deleted_keys == []