LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
adapter data migration for vecto db adapter (#376)
Rahul Johnya43dfafd780e
diff --git a/backend/adapter_processor/migrations/0009_frictionless_adapter_data_migration.py b/backend/adapter_processor/migrations/0009_frictionless_adapter_data_migration.pynew file mode 100644index 000000000..e4eb37167--- /dev/null+++ b/backend/adapter_processor/migrations/0009_frictionless_adapter_data_migration.py@@ -0,0 +1,71 @@+# Generated by Django 4.2.1 on 2024-06-02 04:17++import json+import logging+from typing import Any++from adapter_processor.constants import AdapterKeys+from cryptography.fernet import Fernet+from django.conf import settings+from django.db import migrations++logger = logging.getLogger(__name__)+++class Migration(migrations.Migration):++ dependencies = [+ ("adapter_processor", "0008_adapterinstance_description_and_more"),+ ]++ def migrate_frictionless_adapter_data(apps: Any, schema_editor: Any) -> None:+ """Migrates the data for the frictionless adapter by encrypting and+ saving the adapter metadata in the database.++ Parameters:+ apps (Any): The registry of installed applications.+ schema_editor (Any): The schema editor for the database operation.++ Returns:+ None: This method does not return anything.++ Raises:+ None: This method does not raise any exceptions.+ """+ VECTOR_DB_CONF = "VECTOR_DB_CONF"++ # check vector db conf is added in env+ if not hasattr(settings, VECTOR_DB_CONF):+ return++ AdapterInstance = apps.get_model("adapter_processor", "AdapterInstance")++ try:+ adapter_instance = AdapterInstance.objects.get(+ is_friction_less=True,+ is_usable=True,+ adapter_type=AdapterKeys.VECTOR_DB,+ )++ vector_db_conf = json.loads(getattr(settings, VECTOR_DB_CONF))+ adapter_metadata = vector_db_conf["adapter_metadata"]++ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))++ adapter_metadata_b: bytes = f.encrypt(+ json.dumps(adapter_metadata).encode("utf-8")+ )++ adapter_instance.adapter_metadata_b = adapter_metadata_b+ adapter_instance.save()++ except AdapterInstance.DoesNotExist:+ logger.info("Skip data migration as frictionless vector-db not available")++ operations = [+ migrations.RunPython(+ migrate_frictionless_adapter_data,+ reverse_code=migrations.RunPython.noop,+ ),+ ]