LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Moved away from storing encryption key in database to ENV (#39)
Rahul Johnyb8be52176cce
diff --git a/README.md b/README.mdindex 565e84400..bf0d32ff9 100644--- a/README.md+++ b/README.md@@ -176,6 +176,14 @@ We just need to override default Traefik proxy routing to allow this, that's allRun the services.+#### Generate Encryption key to be used in backend and Platform service++ Generate Fernet Key Refer https://pypi.org/project/cryptography/++ `ENCRYPTION_KEY=$(python -c "import secrets, base64; print(base64.urlsafe_b64encode(secrets.token_bytes(32)).decode())")`++ use the above generated encryption, key in ENV's of platform and backend+#### Conflicting Host NamesWhen same host name environment variables are used by both the service running locally and a servicediff --git a/backend/account/migrations/0005_encryptionsecret.py b/backend/account/migrations/0005_encryptionsecret.pyindex d8ff25b8a..724c12522 100644--- a/backend/account/migrations/0005_encryptionsecret.py+++ b/backend/account/migrations/0005_encryptionsecret.py@@ -1,9 +1,5 @@# Generated by Django 4.2.1 on 2024-02-13 11:52-from typing import Any--from account.models import EncryptionSecret-from cryptography.fernet import Fernetfrom django.db import migrations, models@@ -12,11 +8,6 @@ class Migration(migrations.Migration):("account", "0004_alter_platformkey_key_name_and_more"),]- def initialize_secret(apps: Any, schema_editor: Any) -> None:- EncryptionSecret.objects.create(- key=Fernet.generate_key().decode("utf-8")- )-operations = [migrations.CreateModel(name="EncryptionSecret",@@ -33,7 +24,4 @@ class Migration(migrations.Migration):("key", models.CharField(blank=True, max_length=64)),],),- migrations.RunPython(- initialize_secret, reverse_code=migrations.RunPython.noop- ),]diff --git a/backend/account/migrations/0006_delete_encryptionsecret.py b/backend/account/migrations/0006_delete_encryptionsecret.pynew file mode 100644index 000000000..1216373ee--- /dev/null+++ b/backend/account/migrations/0006_delete_encryptionsecret.py@@ -0,0 +1,15 @@+# Generated by Django 4.2.1 on 2024-03-04 05:06++from django.db import migrations+++class Migration(migrations.Migration):+ dependencies = [+ ("account", "0005_encryptionsecret"),+ ]++ operations = [+ migrations.DeleteModel(+ name="EncryptionSecret",+ ),+ ]diff --git a/backend/account/models.py b/backend/account/models.pyindex c3bafc8fb..63dade30d 100644--- a/backend/account/models.py+++ b/backend/account/models.py@@ -1,10 +1,11 @@import uuid-from backend.constants import FieldLengthConstants as FieldLengthfrom django.contrib.auth.models import AbstractUser, Group, Permissionfrom django.db import modelsfrom django_tenants.models import DomainMixin, TenantMixin+from backend.constants import FieldLengthConstants as FieldLength+NAME_SIZE = 64KEY_SIZE = 64@@ -131,11 +132,3 @@ class PlatformKey(models.Model):name="unique_key_name",),]---class EncryptionSecret(models.Model):- key = models.CharField(- max_length=KEY_SIZE,- null=False,- blank=True,- )diff --git a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.pyindex 5f65c0040..d5a9bb4cf 100644--- a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py+++ b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py@@ -3,21 +3,20 @@import jsonfrom typing import Any-from account.models import EncryptionSecretfrom adapter_processor.models import AdapterInstancefrom cryptography.fernet import Fernet+from django.conf import settingsfrom django.db import migrations, modelsclass Migration(migrations.Migration):dependencies = [("adapter_processor", "0002_adapterinstance_unique_adapter"),- ("account", "0005_encryptionsecret"),]def EncryptCredentials(apps: Any, schema_editor: Any) -> None:- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))queryset = AdapterInstance.objects.all()for obj in queryset: # type: ignorediff --git a/backend/adapter_processor/serializers.py b/backend/adapter_processor/serializers.pyindex bb2443450..577cd5ffb 100644--- a/backend/adapter_processor/serializers.py+++ b/backend/adapter_processor/serializers.py@@ -1,10 +1,10 @@import jsonfrom typing import Any-from account.models import EncryptionSecretfrom adapter_processor.adapter_processor import AdapterProcessorfrom adapter_processor.constants import AdapterKeysfrom cryptography.fernet import Fernet+from django.conf import settingsfrom rest_framework import serializersfrom unstract.adapters.constants import Common as common@@ -45,8 +45,8 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):"""def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]:- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))json_string: str = json.dumps(data.pop(AdapterKeys.ADAPTER_METADATA))data[AdapterKeys.ADAPTER_METADATA_B] = f.encrypt(@@ -58,8 +58,8 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):def to_representation(self, instance: AdapterInstance) -> dict[str, str]:rep: dict[str, str] = super().to_representation(instance)- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))rep.pop(AdapterKeys.ADAPTER_METADATA_B)adapter_metadata = json.loads(diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 4252b1483..9b9deea2f 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -143,6 +143,7 @@ API_STORAGE_DIR = os.environ.get("API_STORAGE_DIR")# SECURITY WARNING: keep the secret key used in production secret!SECRET_KEY = get_required_setting("DJANGO_SECRET_KEY")+ENCRYPTION_KEY = get_required_setting("ENCRYPTION_KEY")# SECURITY WARNING: don't run with debug turned on in production!DEBUG = Truediff --git a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.pyindex 7f9090a2e..1e4a342c9 100644--- a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py+++ b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py@@ -3,21 +3,20 @@import jsonfrom typing import Any-from account.models import EncryptionSecretfrom connector.models import ConnectorInstancefrom cryptography.fernet import Fernet+from django.conf import settingsfrom django.db import migrations, modelsclass Migration(migrations.Migration):dependencies = [("connector", "0001_initial"),- ("account", "0005_encryptionsecret"),]def EncryptCredentials(apps: Any, schema_editor: Any) -> None:- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))queryset = ConnectorInstance.objects.all()for obj in queryset: # type: ignorediff --git a/backend/connector/models.py b/backend/connector/models.pyindex bc37af041..eb0a3f7da 100644--- a/backend/connector/models.py+++ b/backend/connector/models.py@@ -2,12 +2,13 @@ import jsonimport uuidfrom typing import Any-from account.models import EncryptionSecret, User+from account.models import Userfrom connector.fields import ConnectorAuthJSONFieldfrom connector_auth.models import ConnectorAuthfrom connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeysfrom cryptography.fernet import Fernet+from django.conf import settingsfrom django.db import modelsfrom project.models import Projectfrom utils.models.base_model import BaseModel@@ -112,8 +113,8 @@ class ConnectorInstance(BaseModel):@propertydef metadata(self) -> Any:- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()- cipher_suite: Fernet = Fernet(encryption_secret.key.encode("utf-8"))+ encryption_secret: str = settings.ENCRYPTION_KEY+ cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))decrypted_value = cipher_suite.decrypt(bytes(self.connector_metadata_b).decode("utf-8"))diff --git a/backend/connector/serializers.py b/backend/connector/serializers.pyindex 4b664cb67..820b47d66 100644--- a/backend/connector/serializers.py+++ b/backend/connector/serializers.py@@ -3,7 +3,6 @@ import loggingfrom collections import OrderedDictfrom typing import Any, Optional-from account.models import EncryptionSecretfrom connector.constants import ConnectorInstanceKey as CIKeyfrom connector_auth.models import ConnectorAuthfrom connector_auth.pipeline.common import ConnectorAuthHelper@@ -11,6 +10,7 @@ from connector_processor.connector_processor import ConnectorProcessorfrom connector_processor.constants import ConnectorKeysfrom connector_processor.exceptions import OAuthTimeOutfrom cryptography.fernet import Fernet+from django.conf import settingsfrom utils.serializer_utils import SerializerUtilsfrom backend.serializers import AuditSerializer@@ -55,8 +55,8 @@ class ConnectorInstanceSerializer(AuditSerializer):)kwargs[CIKey.CONNECTOR_MODE] = connector_mode.value- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))json_string: str = json.dumps(kwargs.pop(CIKey.CONNECTOR_METADATA))if self.validated_data:self.validated_data.pop(CIKey.CONNECTOR_METADATA)@@ -81,8 +81,8 @@ class ConnectorInstanceSerializer(AuditSerializer):] = ConnectorProcessor.get_connector_data_with_key(instance.connector_id, ConnectorKeys.ICON)- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))+ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))rep.pop(CIKey.CONNECTOR_METADATA_B)if instance.connector_metadata_b:diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.pyindex 25f4b215b..150d97371 100644--- a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py@@ -4,15 +4,16 @@ import jsonimport django.db.models.deletionfrom cryptography.fernet import Fernet+from django.conf import settingsfrom django.db import connection, migrations, modelsdef fill_with_default_x2text(apps, schema):ProfileManager = apps.get_model("prompt_profile_manager", "ProfileManager")AdapterInstance = apps.get_model("adapter_processor", "AdapterInstance")- EncryptionSecret = apps.get_model("account", "EncryptionSecret")- encryption_secret = EncryptionSecret.objects.get()- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))++ encryption_secret: str = settings.ENCRYPTION_KEY+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))metadata = {"url": "http://unstract-unstructured-io:8000/general/v0/general"}@@ -44,7 +45,6 @@ def disable_triggers(apps, schema_editor):class Migration(migrations.Migration):dependencies = [- ("account", "0005_encryptionsecret"),("adapter_processor", "0004_alter_adapterinstance_adapter_type"),("prompt_profile_manager",diff --git a/backend/sample.env b/backend/sample.envindex 1ffd6a168..5fca2ee4d 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -100,3 +100,7 @@ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python#X2Text ServiceX2TEXT_HOST=http://unstract-x2text-serviceX2TEXT_PORT=3004++# Encryption Key+# key must be 32 url-safe base64-encoded bytes.+ENCRYPTION_KEY="Sample-Key"diff --git a/platform-service/sample.env b/platform-service/sample.envindex 03b2a1051..3d8a69a13 100644--- a/platform-service/sample.env+++ b/platform-service/sample.env@@ -23,3 +23,7 @@ PG_V_PORT=5432PG_V_USERNAME=unstract_devPG_V_PASSWORD=unstract_passPG_V_DATABASE=unstract_db++# Encryption Key+# key must be 32 url-safe base64-encoded bytes.+ENCRYPTION_KEY="Sample-Key"diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.pyindex 8aad19928..627343055 100644--- a/platform-service/src/unstract/platform_service/main.py+++ b/platform-service/src/unstract/platform_service/main.py@@ -45,6 +45,7 @@ PG_V_PORT = int(os.environ.get("PG_V_PORT", 5432))PG_V_USERNAME = os.environ.get("PG_V_USERNAME", "user")PG_V_PASSWORD = os.environ.get("PG_V_PASSWORD", "")PG_V_DATABASE = os.environ.get("PG_V_DATABASE", "")+ENCRYPTION_KEY = os.environ.get("ENCRYPTION_KEY")if not (REDIS_HOST and REDIS_PORT):raise ValueError("REDIS_HOST and REDIS_PORT must be set in the environment."@@ -74,19 +75,6 @@ be_db.init(PG_BE_DATABASE)be_db.connect()-class EncryptionSecret(peewee.Model):- id = peewee.IntegerField()- key = peewee.CharField()-- class Meta:- table_name = "account_encryptionsecret"- database = be_db # This model uses the "BE_DB" database.-- def save(self, force_insert: bool = False, only: Any = None) -> Any:- self.modified_at = datetime.datetime.now()- return super().save(force_insert=force_insert, only=only)--class UnstractUsage(peewee.Model):id = peewee.UUIDField(primary_key=True, default=uuid.uuid4)created_at = peewee.DateTimeField(default=datetime.datetime.now)@@ -408,8 +396,7 @@ def adapter_instance() -> Any:))- encryption_secret: EncryptionSecret = EncryptionSecret.get()- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))+ f: Fernet = Fernet(ENCRYPTION_KEY.encode("utf-8"))data_dict["adapter_metadata"] = json.loads(f.decrypt(