Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

adapter data migration for vecto db adapter (#376)

Rahul Johnya43dfafd780e
diff --git a/backend/adapter_processor/migrations/0009_frictionless_adapter_data_migration.py b/backend/adapter_processor/migrations/0009_frictionless_adapter_data_migration.py
new file mode 100644
index 000000000..e4eb37167
--- /dev/null
+++ b/backend/adapter_processor/migrations/0009_frictionless_adapter_data_migration.py
@@ -0,0 +1,71 @@
+# Generated by Django 4.2.1 on 2024-06-02 04:17
+
+import json
+import logging
+from typing import Any
+
+from adapter_processor.constants import AdapterKeys
+from cryptography.fernet import Fernet
+from django.conf import settings
+from django.db import migrations
+
+logger = logging.getLogger(__name__)
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("adapter_processor", "0008_adapterinstance_description_and_more"),
+ ]
+
+ def migrate_frictionless_adapter_data(apps: Any, schema_editor: Any) -> None:
+ """Migrates the data for the frictionless adapter by encrypting and
+ saving the adapter metadata in the database.
+
+ Parameters:
+ apps (Any): The registry of installed applications.
+ schema_editor (Any): The schema editor for the database operation.
+
+ Returns:
+ None: This method does not return anything.
+
+ Raises:
+ None: This method does not raise any exceptions.
+ """
+ VECTOR_DB_CONF = "VECTOR_DB_CONF"
+
+ # check vector db conf is added in env
+ if not hasattr(settings, VECTOR_DB_CONF):
+ return
+
+ AdapterInstance = apps.get_model("adapter_processor", "AdapterInstance")
+
+ try:
+ adapter_instance = AdapterInstance.objects.get(
+ is_friction_less=True,
+ is_usable=True,
+ adapter_type=AdapterKeys.VECTOR_DB,
+ )
+
+ vector_db_conf = json.loads(getattr(settings, VECTOR_DB_CONF))
+ adapter_metadata = vector_db_conf["adapter_metadata"]
+
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
+
+ adapter_metadata_b: bytes = f.encrypt(
+ json.dumps(adapter_metadata).encode("utf-8")
+ )
+
+ adapter_instance.adapter_metadata_b = adapter_metadata_b
+ adapter_instance.save()
+
+ except AdapterInstance.DoesNotExist:
+ logger.info("Skip data migration as frictionless vector-db not available")
+
+ operations = [
+ migrations.RunPython(
+ migrate_frictionless_adapter_data,
+ reverse_code=migrations.RunPython.noop,
+ ),
+ ]