LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
data migrations for v2 applications (#537)
alia2751f0dd827
diff --git a/backend/migrating/v2/README.md b/backend/migrating/v2/README.mdnew file mode 100644index 000000000..3c5867d35--- /dev/null+++ b/backend/migrating/v2/README.md@@ -0,0 +1,59 @@+# V2 Migrations and Commands++This folder contains scripts and management commands for performing data migrations from v1 to v2 for the Unstract Multitenancy system.++## Overview++The commands provided here help you manage the migration process, including creating and dropping schemas and applying the necessary migrations to transition from v1 to v2.+++**Pre-requisites:**+Ensure that `migrating.v2` is added to the `SHARED_APPS` in your Django settings.++## Available Commands++### 1. List Commands++To view all available management commands, use:++```bash+python manage.py help+```++### 2. Create V2 Schema++To set up the v2 schema in your database, execute:++```bash+python manage.py create_v2_schema+```+This command initializes the schema required for v2 migrations.++### 3. Drop V2 Schema++To remove the v2 schema from your database, run:++```bash+python manage.py drop_v2_schema+```+Note: Use this command with caution as it will permanently delete the v2 schema and its associated data.++### 4. Migrate to V2 Schema+To apply migrations and transition data from v1 to v2, use:++```bash+python manage.py migrate_to_v2+```++This command performs the migration, updating your database to the v2 schema and transforming the data accordingly.+++## Notes+- Ensure you have backups of your data before running migration commands.+- Verify your database configuration and environment variables are correctly set.+- Review the migration logs for any issues or errors during the migration process.++For more details on each command and its options, refer to the command help output using+```bash+python manage.py help <command_name>+```diff --git a/backend/migrating/v2/__init__.py b/backend/migrating/v2/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/migrating/v2/admin.py b/backend/migrating/v2/admin.pynew file mode 100644index 000000000..846f6b406--- /dev/null+++ b/backend/migrating/v2/admin.py@@ -0,0 +1 @@+# Register your models here.diff --git a/backend/migrating/v2/apps.py b/backend/migrating/v2/apps.pynew file mode 100644index 000000000..cecdfd3cb--- /dev/null+++ b/backend/migrating/v2/apps.py@@ -0,0 +1,6 @@+from django.apps import AppConfig+++class MigrationToolsConfig(AppConfig):+ default_auto_field = "django.db.models.BigAutoField"+ name = "migrating.v2"diff --git a/backend/migrating/v2/constants.py b/backend/migrating/v2/constants.pynew file mode 100644index 000000000..638acb6be--- /dev/null+++ b/backend/migrating/v2/constants.py@@ -0,0 +1,5 @@+import os+++class V2:+ SCHEMA_NAME = os.getenv("DB_SCHEMA", None)diff --git a/backend/migrating/v2/management/__init__.py b/backend/migrating/v2/management/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/migrating/v2/management/commands/__init__.py b/backend/migrating/v2/management/commands/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/migrating/v2/management/commands/create_v2_schema.py b/backend/migrating/v2/management/commands/create_v2_schema.pynew file mode 100644index 000000000..5be10a121--- /dev/null+++ b/backend/migrating/v2/management/commands/create_v2_schema.py@@ -0,0 +1,18 @@+from django.core.management.base import BaseCommand+from django.db import connection+from migrating.v2.constants import V2+++class Command(BaseCommand):+ help = "Create v2 schema if it does not exist"++ def handle(self, *args, **kwargs):+ if not V2.SCHEMA_NAME:+ raise ValueError("SCHEMA_NAME is not defined.")+ with connection.cursor() as cursor:+ cursor.execute(f"CREATE SCHEMA IF NOT EXISTS {V2.SCHEMA_NAME}")+ self.stdout.write(+ self.style.SUCCESS(+ f'Schema "{V2.SCHEMA_NAME}" checked/created successfully.'+ )+ )diff --git a/backend/migrating/v2/management/commands/drop_v2_schema.py b/backend/migrating/v2/management/commands/drop_v2_schema.pynew file mode 100644index 000000000..128a40119--- /dev/null+++ b/backend/migrating/v2/management/commands/drop_v2_schema.py@@ -0,0 +1,16 @@+from django.core.management.base import BaseCommand+from django.db import connection+from migrating.v2.constants import V2+++class Command(BaseCommand):+ help = "Drop v2 schema if it exists"++ def handle(self, *args, **kwargs):+ if not V2.SCHEMA_NAME:+ raise ValueError("SCHEMA_NAME is not defined.")+ with connection.cursor() as cursor:+ cursor.execute(f"DROP SCHEMA IF EXISTS {V2.SCHEMA_NAME} CASCADE")+ self.stdout.write(+ self.style.SUCCESS(f'Schema "{V2.SCHEMA_NAME}" deleted successfully.')+ )diff --git a/backend/migrating/v2/management/commands/migrate_to_v2.py b/backend/migrating/v2/management/commands/migrate_to_v2.pynew file mode 100644index 000000000..ff07913a2--- /dev/null+++ b/backend/migrating/v2/management/commands/migrate_to_v2.py@@ -0,0 +1,463 @@+import json+import logging+import os+import time+from collections.abc import Generator+from contextlib import contextmanager+from typing import Any, Optional++import psycopg2+from django.conf import settings+from django.core.management.base import BaseCommand+from migrating.v2.constants import V2+from migrating.v2.unstract_migrations import UnstractMigration+from psycopg2.extensions import connection, cursor++logger = logging.getLogger(__name__)+++class DataMigrator:+ def __init__(self, src_db_config, dest_db_config, v2_schema, batch_size=1000):+ self.src_db_config = src_db_config+ self.dest_db_config = dest_db_config+ self.batch_size = batch_size+ self.v2_schema = v2_schema+ self.migration_tracking_table = f"{self.v2_schema}.migration_tracking"++ @contextmanager+ def _db_connect_and_cursor(+ self, db_config: dict+ ) -> Generator[tuple[connection, cursor], None, None]:+ """A context manager to manage database connection and cursor.++ Args:+ db_config (dict): Database configuration dictionary.++ Yields:+ tuple[connection, cursor]: A tuple containing the connection and cursor.+ """+ conn: connection = psycopg2.connect(**db_config)+ try:+ with conn.cursor() as cur: # Manage cursor within the same context+ yield conn, cur+ except psycopg2.Error as e:+ conn.rollback() # Rollback on error+ logger.error(f"Database operation error: {e}")+ raise+ except Exception as e:+ conn.rollback() # Rollback on unexpected error+ logger.error(f"Unexpected error: {e}")+ raise+ finally:+ conn.close()++ def _create_tracking_table_if_not_exists(self):+ """Ensures that the migration tracking table exists in the destination+ database.++ Creates the table if it does not exist.+ """+ with self._db_connect_and_cursor(self.dest_db_config) as (conn, cur):+ try:+ cur.execute(+ f"""+ CREATE TABLE IF NOT EXISTS {self.migration_tracking_table} (+ id SERIAL PRIMARY KEY,+ migration_name VARCHAR(255) UNIQUE NOT NULL,+ applied_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP+ );+ """+ )+ conn.commit()+ logger.info("Migration tracking table ensured.")+ except psycopg2.Error as e:+ logger.error(f"Error creating migration tracking table: {e}")+ raise++ def _is_migration_applied(self, migration_name):+ """Checks whether the specified migration has already been applied.++ Args:+ migration_name (str): The name of the migration to check.++ Returns:+ bool: True if the migration has been applied, False otherwise.+ """+ with self._db_connect_and_cursor(self.dest_db_config) as (conn, cur):+ cur.execute(+ f"""+ SELECT COUNT(*) FROM {self.migration_tracking_table}+ WHERE migration_name = %s;"+ """,+ (migration_name,),+ )+ result = cur.fetchone()+ return result[0] > 0++ def _record_migration(self, migration_name):+ """Records the completion of a migration by inserting its name into the+ tracking table.++ Args:+ migration_name (str): The name of the migration to record.+ """+ with self._db_connect_and_cursor(self.dest_db_config) as (conn, cur):+ cur.execute(+ f"""+ INSERT INTO {self.migration_tracking_table} (migration_name)+ VALUES (%s);+ """,+ (migration_name,),+ )+ conn.commit()++ def _fetch_schema_names(+ self, schemas_to_migrate: list[str]+ ) -> list[tuple[int, str]]:+ """Fetches schema names and their IDs from the destination database+ based on the provided schema list. Supports fetching all schemas if+ '_ALL_' is specified.++ Args:+ schemas_to_migrate (list[str]): A list of schema names to migrate,+ or '_ALL_' to migrate all.++ Returns:+ list[tuple[int, str]]: A list of tuples containing the ID and schema name.+ """+ with self._db_connect_and_cursor(self.dest_db_config) as (conn, cur):++ # Process schemas_to_migrate: trim spaces and remove empty entries+ schemas_to_migrate = [+ schema.strip() for schema in schemas_to_migrate if schema.strip()+ ]++ # If _ALL_ is present, fetch all schema names+ if schemas_to_migrate == ["_ALL_"]:+ query = "SELECT id, schema_name FROM account_organization;"+ cur.execute(query)+ else:+ placeholders = ",".join(["%s"] * len(schemas_to_migrate))+ query = (+ "SELECT id, schema_name FROM account_organization WHERE "+ f"schema_name IN ({placeholders});"+ )+ cur.execute(query, schemas_to_migrate)++ schema_names = [(row[0], row[1]) for row in cur.fetchall()]+ return schema_names++ def _prepare_row_and_migrate_relations(+ self,+ row: tuple[Any, ...],+ dest_cursor: cursor,+ column_names: list[str],+ column_transformations: dict[str, dict[str, Any]],+ ) -> tuple[Any, ...]:+ """Prepares and migrates the relational keys of a single row from the+ source database to the destination database, updating specific column+ values based on provided transformations.++ Args:+ row (tuple[Any, ...]): The row containing the original relational keys.+ dest_cursor (cursor): Cursor for the destination database.+ column_names (list[str]): List of column names in the row.+ column_transformations (dict[str, dict[str, Any]]):+ A mapping that defines how to transform+ specific column values (usually foreign keys).+ The key is the name of the column to be+ transformed, and the value is a dictionary containing:+ - query (str): SQL query to fetch the corresponding new value.+ - params (list[str]): List of column names to use as+ parameters for the query.+ This is used to migrate old column names from V1 to new column+ names in V2.++ Returns:+ tuple[Any, ...]: The row with updated relational keys.+ """+ row = list(row)++ for key, transaction in column_transformations.items():+ if key in column_names:+ query = transaction["query"]+ params = transaction["params"]+ param_values = [row[column_names.index(param)] for param in params]++ dest_cursor.execute(query, param_values)+ new_key_value = dest_cursor.fetchone()+ if new_key_value:+ row[column_names.index(key)] = new_key_value[0]++ for i, value in enumerate(row):+ if isinstance(value, dict):+ row[i] = json.dumps(value)+ return tuple(row)++ def _migrate_rows(+ self,+ migration_name: str,+ src_cursor: cursor,+ dest_cursor: cursor,+ dest_conn: connection,+ dest_query: str,+ column_names: list[str],+ column_transformations: dict[str, dict[str, Any]],+ ) -> None:+ """Migrates rows in batches from the source to the destination+ database.++ Args:+ migration_name (str): The name of the migration.+ src_cursor (cursor): Cursor for the source database.+ dest_cursor (cursor): Cursor for the destination database.+ dest_conn (connection): Connection to the destination database.+ dest_query (str): SQL query to insert rows into the destination database.+ column_names (list[str]): List of column names in the source data.+ column_transformations (dict[str, dict[str, Any]]):+ A mapping that defines key transformations.+ The key is the name of the column to be transformed, and+ the value is a dictionary containing:+ - "query" (str): SQL query to fetch the new value.+ - "params" (list[str]): List of column names to use as parameters+ for the query.+ This is used to migrate old column names from V1 to new+ column names in V2.++ Returns:+ None+ """+ batch_count = 0+ total_rows = 0++ rows = src_cursor.fetchmany(self.batch_size)+ while rows:+ batch_count += 1+ total_rows += len(rows)+ logger.info(+ f"[{migration_name}] Processing batch {batch_count}: {len(rows)} rows"+ )++ converted_rows = []+ for row in rows:+ converted_row = self._prepare_row_and_migrate_relations(+ row, dest_cursor, column_names, column_transformations+ )+ converted_rows.append(converted_row)++ start_time = time.time()+ try:+ dest_cursor.executemany(dest_query, converted_rows)+ dest_conn.commit()+ end_time = time.time()+ elapsed_time = end_time - start_time+ logger.info(+ f"[{migration_name}] Batch {batch_count} inserted successfully."+ f" Time taken: {elapsed_time:.2f} seconds."+ )+ except psycopg2.Error as e:+ logger.error(+ f"[{migration_name}] Error inserting batch {batch_count}: {e}"+ )+ raise+ # Fetch the next batch of rows+ rows = src_cursor.fetchmany(self.batch_size)+ logger.info(+ f"[{migration_name}] Completed migration with {total_rows} rows processed."+ )++ def _bump_auto_increment_id(+ self, dest_cursor: cursor, dest_table: str, dest_conn: connection+ ) -> None:+ """Adjust the auto-increment ID for a table in the destination+ database.++ This method retrieves the maximum ID value from the specified table+ and sets the next auto-increment value for the table accordingly.+ Args:+ dest_cursor (cursor): The cursor for the destination database.+ dest_table (str): The name of the table to adjust.+ dest_conn (connection): The connection to the destination database.+ Returns:+ None+ """+ try:+ column_type_query = f"""+ SELECT data_type+ FROM information_schema.columns+ WHERE table_schema = '{self.v2_schema}' AND table_name = '{dest_table}'+ AND column_name = 'id';+ """++ dest_cursor.execute(column_type_query)+ result = dest_cursor.fetchone()++ if not result:+ logger.warning(+ f"No 'id' column found in table '{dest_table}' in schema "+ f"'{self.v2_schema}'. Skipping auto-increment adjustment."+ )+ return++ column_type = result[0]+ if column_type not in ("bigint", "integer", "serial"):+ logger.warning(+ f"Column 'id' in table '{dest_table}' is not of a supported "+ "type for auto-increment adjustment."+ )+ return++ max_id_query = f'SELECT MAX(id) FROM "{self.v2_schema}".{dest_table};'+ dest_cursor.execute(max_id_query)+ max_id = dest_cursor.fetchone()[0]++ if max_id is None:+ logger.info(+ f"Table '{dest_table}' is empty. No need to adjust "+ "auto-increment."+ )+ return++ seq_name = f'"{self.v2_schema}".{dest_table}_id_seq'+ seq_query = f"SELECT setval('{seq_name}', {max_id});"+ dest_cursor.execute(seq_query)+ dest_conn.commit()+ logger.info(+ f"Adjusted auto-increment for table {dest_table} "+ f"to start from {max_id + 1}."+ )++ except psycopg2.Error as e:+ logger.error(f"Error adjusting auto-increment for table {dest_table}: {e}")+ raise++ def migrate(+ self, migrations: list[dict[str, str]], organization_id: Optional[str] = None+ ) -> None:+ self._create_tracking_table_if_not_exists()++ with (+ self._db_connect_and_cursor(self.src_db_config) as (src_conn, src_cursor),+ self._db_connect_and_cursor(self.dest_db_config) as (+ dest_conn,+ dest_cursor,+ ),+ ):++ for migration in migrations:+ migration_name = migration["name"]+ logger.info(f"Migration '{migration_name}' started")+ if self._is_migration_applied(migration_name):+ logger.info(+ f"Migration '{migration_name}' has already been applied. "+ "Skipping..."+ )+ continue++ src_query = migration["src_query"]+ dest_query = migration["dest_query"]+ dest_table = migration.get("dest_table")+ clear_table = migration.get("clear_table", False)+ column_transformations = migration.get("new_key_transaction", {})++ # Clear the destination table if specified+ if clear_table and dest_table:+ logger.info(f"Clearing table '{dest_table}'")+ try:+ start_time = time.time()+ dest_cursor.execute(f"DELETE FROM {dest_table}")+ dest_conn.commit()+ elapsed_time = time.time() - start_time+ logger.info(+ f"Table '{dest_table}' cleared successfully. Time taken: "+ f"{elapsed_time:.2f} seconds."+ )+ except Exception as e:+ logger.error(f"Error clearing table '{dest_table}': {e}")+ raise++ src_cursor.execute(src_query)+ column_names = [desc[0] for desc in src_cursor.description]++ self._migrate_rows(+ migration_name,+ src_cursor,+ dest_cursor,+ dest_conn,+ dest_query,+ column_names,+ column_transformations,+ )++ # Adjust the auto-increment value+ if dest_table and "id" in column_names:+ self._bump_auto_increment_id(dest_cursor, dest_table, dest_conn)+ dest_conn.commit()++ # Record the migration+ self._record_migration(migration_name)+ logger.info(f"Migration '{migration_name}' completed successfully")+++class Command(BaseCommand):+ help = (+ "Migrate data from v1 models to v2 models across all applications.\n\n"+ "This command facilitates data migration for the specified schemas or "+ "for all schemas if '_ALL_' is provided. It connects to both the source "+ "and destination databases and ensures the data is properly migrated "+ "from the v1 schema to the new v2 schema."+ )++ if not V2.SCHEMA_NAME:+ raise ValueError("SCHEMA_NAME is not defined.")++ def handle(self, *args, **options):+ v2_schema = V2.SCHEMA_NAME+ src_db_config = {+ "dbname": settings.DB_NAME,+ "user": settings.DB_USER,+ "password": settings.DB_PASSWORD,+ "host": settings.DB_HOST,+ "port": settings.DB_PORT,+ }+ dest_db_config = {+ "dbname": settings.DB_NAME,+ "user": settings.DB_USER,+ "password": settings.DB_PASSWORD,+ "host": settings.DB_HOST,+ "port": settings.DB_PORT,+ }+ schemas_to_migrate = os.getenv("SCHEMAS_TO_MIGRATE", None)+ migration_query = UnstractMigration(v2_schema=v2_schema)++ # Public tables+ public_schema_migrations = migration_query.get_public_schema_migrations()+ migrator = DataMigrator(+ src_db_config, dest_db_config, v2_schema, batch_size=1000+ )+ migrator.migrate(public_schema_migrations)++ if not schemas_to_migrate:+ logger.info("Migration not run since SCHEMAS_TO_MIGRATE env seems empty.")+ return+ else:+ schemas_to_migrate = schemas_to_migrate.split(",")++ # Organization Data (Schema)+ schema_names = migrator._fetch_schema_names(+ schemas_to_migrate=schemas_to_migrate+ )++ for organization_id, schema_name in schema_names:+ if schema_name == "public":+ logger.info("Skipping public schema migration")+ continue+ logger.info(+ f"Migration started for schema_name {schema_name} ID {organization_id}"+ )+ migrations = migration_query.get_organization_migrations(+ schema=schema_name, organization_id=organization_id+ )+ migrator.migrate(migrations, organization_id)+ completion_message = "DATA MIGRATION COMPLETED SUCCESSFULLY !!!"+ logger.info(completion_message)diff --git a/backend/migrating/v2/migrations/__init__.py b/backend/migrating/v2/migrations/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/migrating/v2/query.py b/backend/migrating/v2/query.pynew file mode 100644index 000000000..fc39bd740--- /dev/null+++ b/backend/migrating/v2/query.py@@ -0,0 +1,696 @@+class MigrationQuery:+ """This class contains methods to generate SQL queries for various+ migration operations."""++ def __init__(self, v2_schema) -> None:+ self.v2_schema = v2_schema++ def get_public_schema_migrations(self) -> list[dict[str, str]]:+ """Returns a list of dictionaries containing the schema migration+ details.++ Args:+ v2_schema (str): The name of the schema in the version 2 database.++ Returns:+ list: A list of dictionaries containing the schema migration details.+ """+ migrations = [+ {+ "name": "001_organization",+ "src_query": """+ SELECT id, name, display_name, organization_id, modified_at,+ created_at, allowed_token_limit FROM account_organization;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".organization (id, name,+ display_name, organization_id, modified_at, created_at,+ allowed_token_limit)+ VALUES (%s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "organization",+ },+ {+ "name": "migration_002_user",+ "src_query": """+ SELECT id, password, last_login, is_superuser, username,+ first_name, last_name, email, is_staff, is_active,+ date_joined, user_id, project_storage_created, created_by_id,+ modified_by_id, modified_at, created_at+ FROM account_user;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".user (+ id, password, last_login, is_superuser, username, first_name,+ last_name, email, is_staff, is_active, date_joined, user_id,+ project_storage_created, created_by_id, modified_by_id,+ modified_at, created_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, %s, %s);+ """,+ "dest_table": "user",+ },+ {+ "name": "migration_003_platformkey",+ "src_query": """+ SELECT id, key, key_name, is_active, organization_id,+ created_by_id, modified_by_id+ FROM account_platformkey;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".platform_key (+ id, key, key_name, is_active, organization_id,+ created_by_id, modified_by_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "platform_key",+ },+ {+ "name": "migration_004_periodictask",+ "src_query": """+ SELECT id, name, task, args, kwargs, queue,+ exchange, routing_key, expires, enabled, last_run_at,+ total_run_count, date_changed,+ description, crontab_id, interval_id, solar_id,+ one_off, start_time, priority, headers,+ clocked_id, expire_seconds+ FROM django_celery_beat_periodictask;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".django_celery_beat_periodictask (+ id, name, task, args, kwargs, queue, exchange, routing_key,+ expires, enabled, last_run_at,+ total_run_count, date_changed, description, crontab_id,+ interval_id, solar_id, one_off,+ start_time, priority, headers, clocked_id, expire_seconds+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);+ """,+ "clear_table": True,+ "dest_table": "django_celery_beat_periodictask",+ },+ {+ "name": "migration_005_periodictasks",+ "src_query": """+ SELECT ident, last_update FROM django_celery_beat_periodictasks;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".django_celery_beat_periodictasks+ (ident, last_update)+ VALUES (%s, %s);+ """,+ "clear_table": True,+ "dest_table": "django_celery_beat_periodictasks",+ },+ {+ "name": "migration_006_crontabschedule",+ "src_query": """+ SELECT id, minute, hour, day_of_week, day_of_month,+ month_of_year, timezone FROM django_celery_beat_crontabschedule;+ """,+ "dest_query": f"""+ INSERT INTO+ "{self.v2_schema}".django_celery_beat_crontabschedule (+ id, minute, hour, day_of_week, day_of_month, month_of_year,+ timezone+ ) VALUES (%s, %s, %s, %s, %s, %s, %s);+ """,+ "clear_table": True,+ "dest_table": "django_celery_beat_crontabschedule",+ },+ {+ "name": "migration_007_intervalschedule",+ "src_query": """+ SELECT id, every, period FROM django_celery_beat_intervalschedule;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".django_celery_beat_intervalschedule+ (id, every, period)+ VALUES (%s, %s, %s);+ """,+ "clear_table": True,+ "dest_table": "django_celery_beat_intervalschedule",+ },+ {+ "name": "migration_008_clockedschedule",+ "src_query": """+ SELECT id, clocked_time FROM django_celery_beat_clockedschedule;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".django_celery_beat_clockedschedule+ (id, clocked_time)+ VALUES (%s, %s);+ """,+ "clear_table": True,+ "dest_table": "django_celery_beat_clockedschedule",+ },+ {+ "name": "migration_009_djangosession",+ "src_query": """+ SELECT session_key, session_data, expire_date FROM django_session;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".django_session (+ session_key, session_data, expire_date+ ) VALUES (%s, %s, %s);+ """,+ "dest_table": "django_session",+ },+ {+ "name": "migration_010_socialauth_usersocialauth",+ "src_query": """+ SELECT id, user_id, provider, uid, extra_data, created, modified+ FROM social_auth_usersocialauth;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".social_auth_usersocialauth (+ id, user_id, provider, uid, extra_data, created, modified+ ) VALUES (%s, %s, %s, %s, %s);+ """,+ "dest_table": "social_auth_usersocialauth",+ },+ {+ "name": "migration_011_socialauth_nonce",+ "src_query": """+ SELECT id, server_url, timestamp, salt FROM social_auth_nonce;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".social_auth_nonce (+ id, server_url, timestamp, salt+ ) VALUES (%s, %s, %s, %s);+ """,+ "dest_table": "social_auth_nonce",+ },+ {+ "name": "migration_012_socialauth_association",+ "src_query": """+ SELECT id, server_url, handle, secret, issued, lifetime,+ assoc_type FROM social_auth_association;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".social_auth_association (+ id, server_url, handle, secret, issued, lifetime, assoc_type+ ) VALUES (%s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "social_auth_association",+ },+ {+ "name": "migration_013_socialauth_code",+ "src_query": """+ SELECT id, email, code, verified, timestamp FROM social_auth_code;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".social_auth_code (+ id, email, code, verified, timestamp+ ) VALUES (%s, %s, %s, %s, %s);+ """,+ "dest_table": "social_auth_code",+ },+ {+ "name": "migration_014_socialauth_partial",+ "src_query": """+ SELECT id, token, data, next_step, backend, timestamp+ FROM social_auth_partial;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".social_auth_partial (+ id, token, data, next_step, backend, timestamp+ ) VALUES (%s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "social_auth_partial",+ },+ {+ "name": "migration_016_connector_auth",+ "src_query": """+ SELECT id, user_id, provider, uid, extra_data, created, modified+ FROM connector_auth_connectorauth;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".connector_auth (+ id, user_id, provider, uid, extra_data, created, modified+ ) VALUES (%s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "connector_auth",+ },+ ]+ return migrations++ def get_organization_migrations(+ self, schema: str, organization_id: str+ ) -> list[dict[str, str]]:+ """+ Returns a list of dictionaries containing the organization migration details.+ Args:+ schema (str): The name of the schema for the organization.+ organization_id (str): The ID of the organization.++ Returns:+ list: A list of dictionaries containing the organization migration+ details.+ """+ migrations = [+ {+ "name": f"migration_{schema}_001_organization_member",+ "src_query": f"""+ SELECT user_id, role, is_login_onboarding_msg,+ is_prompt_studio_onboarding_msg+ FROM "{schema}".tenant_account_organizationmember;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".organization_member (+ user_id, role, is_login_onboarding_msg,+ is_prompt_studio_onboarding_msg,+ organization_id+ ) VALUES (%s, %s, %s, %s, {organization_id});+ """,+ "dest_table": "organization_member",+ },+ {+ "name": f"migration_{schema}_002_adapter_instance",+ "src_query": f"""+ SELECT id, adapter_name, adapter_id, adapter_metadata,+ adapter_metadata_b, adapter_type, created_by_id,+ modified_by_id, is_active, shared_to_org, is_friction_less,+ is_usable, description, modified_at, created_at+ FROM "{schema}".adapter_adapterinstance;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".adapter_instance (+ id, adapter_name, adapter_id, adapter_metadata,+ adapter_metadata_b, adapter_type, created_by_id,+ modified_by_id, is_active, shared_to_org, is_friction_less,+ is_usable, description, modified_at, created_at,+ organization_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, %s, {organization_id});+ """,+ "dest_table": "adapter_instance",+ },+ {+ "name": f"migration_{schema}_003_workflow",+ "src_query": f"""+ SELECT id, prompt_name, description, workflow_name, prompt_text,+ is_active, status, llm_response, workflow_owner_id,+ deployment_type, source_settings, destination_settings,+ created_by_id, modified_by_id, modified_at, created_at+ FROM "{schema}".workflow_workflow;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".workflow (+ id, prompt_name, description, workflow_name, prompt_text,+ is_active, status, llm_response, workflow_owner_id,+ deployment_type, source_settings, destination_settings,+ created_by_id, modified_by_id, modified_at, created_at,+ organization_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, %s, %s, {organization_id});+ """,+ "dest_table": "workflow",+ },+ {+ "name": f"migration_{schema}_connector_instance",+ "src_query": f"""+ SELECT id, connector_name, workflow_id, connector_id,+ connector_metadata_b, connector_version, connector_type,+ connector_auth_id, connector_mode, created_by_id,+ modified_by_id, created_at, modified_at+ FROM "{schema}".connector_connectorinstance;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".connector_instance (+ id, connector_name, workflow_id, connector_id,+ connector_metadata, connector_version, connector_type,+ connector_auth_id, connector_mode, created_by_id,+ modified_by_id, created_at, modified_at, organization_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ {organization_id});+ """,+ "dest_table": "connector_instance",+ },+ {+ "name": f"migration_{schema}_workflow_endpoint",+ "src_query": f"""+ SELECT id, workflow_id, endpoint_type, connection_type,+ configuration, connector_instance_id,+ modified_at, created_at+ FROM "{schema}".workflow_endpoints;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".workflow_endpoints (+ id, workflow_id, endpoint_type, connection_type,+ configuration, connector_instance_id,+ modified_at, created_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "workflow_endpoints",+ },+ {+ "name": f"migration_{schema}_api_deployment",+ "src_query": f"""+ SELECT id, display_name, description, workflow_id, is_active,+ api_endpoint, api_name, created_by_id, modified_by_id,+ modified_at, created_at+ FROM "{schema}".api_apideployment;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".api_deployment (+ id, display_name, description, workflow_id, is_active,+ api_endpoint, api_name, created_by_id, modified_by_id,+ modified_at, created_at, organization_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ {organization_id});+ """,+ "dest_table": "api_deployment",+ },+ {+ "name": f"migration_{schema}_api_key",+ "src_query": f"""+ SELECT id, api_key, api_id, description, is_active,+ created_by_id, modified_by_id, created_at, modified_at+ FROM "{schema}".api_apikey;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".api_deployment_key (+ id, api_key, api_id, description, is_active,+ created_by_id, modified_by_id, created_at, modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "api_deployment_key",+ },+ {+ "name": f"migration_{schema}_pipeline",+ "src_query": f"""+ SELECT id, pipeline_name, workflow_id, app_id, active, scheduled,+ cron_string, pipeline_type, run_count,+ last_run_time, last_run_status, app_icon, app_url,+ access_control_bundle_id, created_by_id, modified_by_id,+ created_at, modified_at+ FROM "{schema}".pipeline_pipeline;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".pipeline (+ id, pipeline_name, workflow_id, app_id, active, scheduled,+ cron_string, pipeline_type, run_count,+ last_run_time, last_run_status, app_icon, app_url,+ access_control_bundle_id, created_by_id, modified_by_id,+ created_at, modified_at, organization_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, %s, %s, %s, {organization_id});+ """,+ "dest_table": "pipeline",+ },+ {+ "name": f"migration_{schema}_usage",+ "src_query": f"""+ SELECT id, workflow_id, execution_id, adapter_instance_id, run_id,+ usage_type, llm_usage_reason, model_name,+ embedding_tokens, prompt_tokens, completion_tokens,+ total_tokens, cost_in_dollars,+ created_at, modified_at+ FROM "{schema}".token_usage;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".token_usage (+ id, workflow_id, execution_id, adapter_instance_id, run_id,+ usage_type, llm_usage_reason, model_name,+ embedding_tokens, prompt_tokens, completion_tokens,+ total_tokens, cost_in_dollars,+ created_at, modified_at, organization_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, {organization_id});+ """,+ "dest_table": "token_usage",+ },+ {+ "name": f"migration_{schema}_workflow_execution",+ "src_query": f"""+ SELECT id, pipeline_id, task_id, workflow_id, execution_mode,+ execution_method, execution_type, execution_log_id, status,+ error_message, attempts, execution_time, created_at,+ modified_at+ FROM "{schema}".workflow_workflowexecution;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".workflow_execution (+ id, pipeline_id, task_id, workflow_id, execution_mode,+ execution_method, execution_type, execution_log_id, status,+ error_message, attempts, execution_time, created_at,+ modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "workflow_execution",+ },+ {+ "name": f"migration_{schema}_file_history",+ "src_query": f"""+ SELECT id, cache_key, workflow_id, status, error, result,+ meta_data, created_at, modified_at+ FROM "{schema}".workflow_filehistory;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".file_history (+ id, cache_key, workflow_id, status, error, result, meta_data,+ created_at, modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "file_history",+ },+ {+ "name": f"migration_{schema}_execution_log",+ "src_query": f"""+ SELECT id, execution_id, data, event_time, created_at, modified_at+ FROM "{schema}".workflow_executionlog;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".execution_log (+ id, execution_id, data, event_time, created_at, modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "execution_log",+ },+ {+ "name": f"migration_{schema}_user_default_adapter",+ "src_query": f"""+ SELECT+ user_id,+ default_llm_adapter_id,+ default_embedding_adapter_id,+ default_vector_db_adapter_id,+ default_x2text_adapter_id,+ created_at,+ modified_at+ FROM "{schema}".adapter_processor_userdefaultadapter;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".default_organization_user_adapter (+ organization_member_id,+ default_llm_adapter_id,+ default_embedding_adapter_id,+ default_vector_db_adapter_id,+ default_x2text_adapter_id,+ created_at,+ modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "default_organization_user_adapter",+ "new_key_transaction": {+ "user_id": {+ "query": f"""+ SELECT member_id+ FROM "{self.v2_schema}".organization_member+ WHERE user_id = %s AND organization_id='{organization_id}';+ """,+ "params": ["user_id"],+ }+ },+ },+ # Prompt studio models+ {+ "name": f"migration_{schema}_custom_tool",+ "src_query": f"""+ SELECT tool_id, tool_name, description, author, icon, output,+ log_id, summarize_context, summarize_as_source,+ summarize_prompt, preamble, postamble,+ prompt_grammer, monitor_llm_id, created_by_id,+ modified_by_id, exclude_failed,+ single_pass_extraction_mode, challenge_llm_id,+ enable_challenge, enable_highlight, created_at, modified_at+ FROM "{schema}".prompt_studio_core_customtool;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".custom_tool (+ tool_id, tool_name, description, author, icon, output,+ log_id, summarize_context, summarize_as_source,+ summarize_prompt, preamble, postamble,+ prompt_grammer, monitor_llm_id, created_by_id,+ modified_by_id, exclude_failed,+ single_pass_extraction_mode, challenge_llm_id,+ enable_challenge,+ enable_highlight, created_at, modified_at, organization_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, %s, %s, %s, %s, %s, %s, %s, %s, {organization_id});+ """,+ "dest_table": "custom_tool",+ },+ {+ "name": f"migration_{schema}_custom_tool_shared_users",+ "src_query": f"""+ SELECT customtool_id, user_id+ FROM "{schema}".prompt_studio_core_customtool_shared_users;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".custom_tool_shared_users (+ customtool_id, user_id+ ) VALUES (%s, %s);+ """,+ "dest_table": "custom_tool_shared_users",+ },+ {+ "name": f"migration_{schema}_profile_manager",+ "src_query": f"""+ SELECT profile_id, profile_name, vector_store_id,+ embedding_model_id, llm_id,+ x2text_id, chunk_size, chunk_overlap, reindex,+ retrieval_strategy, similarity_top_k, section, created_by_id,+ modified_by_id, prompt_studio_tool_id, is_default,+ is_summarize_llm, created_at, modified_at+ FROM "{schema}".prompt_profile_manager_profilemanager;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".profile_manager (+ profile_id, profile_name, vector_store_id,+ embedding_model_id, llm_id,+ x2text_id, chunk_size, chunk_overlap, reindex,+ retrieval_strategy, similarity_top_k, section,+ created_by_id, modified_by_id,+ prompt_studio_tool_id, is_default, is_summarize_llm,+ created_at, modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, %s, %s, %s, %s);+ """,+ "dest_table": "profile_manager",+ },+ {+ "name": f"migration_{schema}_document_manager",+ "src_query": f"""+ SELECT document_id, document_name, tool_id, created_by_id,+ modified_by_id, created_at, modified_at+ FROM+ "{schema}".prompt_studio_document_manager_documentmanager;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".document_manager (+ document_id, document_name, tool_id, created_by_id,+ modified_by_id, created_at, modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "document_manager",+ },+ {+ "name": f"migration_{schema}_index_manager",+ "src_query": f"""+ SELECT index_manager_id, document_manager_id, profile_manager_id,+ raw_index_id, summarize_index_id, index_ids_history,+ created_by_id, modified_by_id, created_at, modified_at+ FROM "{schema}".prompt_studio_index_manager_indexmanager;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".index_manager (+ index_manager_id, document_manager_id, profile_manager_id,+ raw_index_id, summarize_index_id, index_ids_history,+ created_by_id, modified_by_id, created_at, modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "index_manager",+ },+ {+ "name": f"migration_{schema}_tool_studio_prompt",+ "src_query": f"""+ SELECT prompt_id, prompt_key, enforce_type, prompt, tool_id_id,+ sequence_number, prompt_type, profile_manager_id, output,+ assert_prompt, assertion_failure_prompt, is_assert, active,+ output_metadata, created_by_id, modified_by_id, evaluate,+ eval_quality_faithfulness, eval_quality_correctness,+ eval_quality_relevance, eval_security_pii,+ eval_guidance_toxicity, eval_guidance_completeness,+ created_at, modified_at+ FROM "{schema}".prompt_studio_toolstudioprompt;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".tool_studio_prompt (+ prompt_id, prompt_key, enforce_type, prompt, tool_id_id,+ sequence_number, prompt_type, profile_manager_id, output,+ assert_prompt, assertion_failure_prompt, is_assert, active,+ output_metadata, created_by_id, modified_by_id, evaluate,+ eval_quality_faithfulness, eval_quality_correctness,+ eval_quality_relevance, eval_security_pii,+ eval_guidance_toxicity, eval_guidance_completeness,+ created_at, modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, %s, %s,+ %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);+ """,+ "dest_table": "tool_studio_prompt",+ },+ {+ "name": f"migration_{schema}_prompt_studio_output_manager",+ "src_query": f"""+ SELECT prompt_output_id, output, context, eval_metrics,+ is_single_pass_extract,+ prompt_id_id, document_manager_id, profile_manager_id,+ tool_id_id, run_id,+ created_by_id, modified_by_id, created_at, modified_at+ FROM+ "{schema}".prompt_studio_output_manager_promptstudiooutputmanager;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".prompt_studio_output_manager (+ prompt_output_id, output, context, eval_metrics,+ is_single_pass_extract,+ prompt_id_id, document_manager_id, profile_manager_id,+ tool_id_id, run_id,+ created_by_id, modified_by_id, created_at, modified_at+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s);+ """,+ "dest_table": "prompt_studio_output_manager",+ },+ {+ "name": f"migration_{schema}_prompt_studio_registry",+ "src_query": f"""+ SELECT prompt_registry_id, name, description, tool_property,+ tool_spec,+ tool_metadata, icon, url, custom_tool_id, created_by_id,+ modified_by_id,+ shared_to_org, created_at, modified_at+ FROM "{schema}".prompt_studio_registry_promptstudioregistry;+ """,+ "dest_query": f"""+ INSERT INTO "{self.v2_schema}".prompt_studio_registry (+ prompt_registry_id, name, description, tool_property,+ tool_spec,+ tool_metadata, icon, url, custom_tool_id, created_by_id,+ modified_by_id,+ shared_to_org, created_at, modified_at, organization_id+ ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,+ %s, {organization_id});+ """,+ "dest_table": "prompt_studio_registry",+ },+ {+ "name": f"migration_{schema}_prompt_studio_registry_shared_users",+ "src_query": f"""+ SELECT promptstudioregistry_id, user_id+ FROM+ "{schema}".prompt_studio_registry_promptstudioregistry_shared_users;+ """,+ "dest_query": f"""+ INSERT INTO+ "{self.v2_schema}".prompt_studio_registry_shared_users (+ promptstudioregistry_id, user_id+ ) VALUES (%s, %s);+ """,+ "dest_table": "prompt_studio_registry_shared_users",+ },+ ]+ return migrationsdiff --git a/backend/migrating/v2/unstract_migrations.py b/backend/migrating/v2/unstract_migrations.pynew file mode 100644index 000000000..7effa6fe0--- /dev/null+++ b/backend/migrating/v2/unstract_migrations.py@@ -0,0 +1,5 @@+from migrating.v2.query import MigrationQuery+++class UnstractMigration(MigrationQuery):+ pass