Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Moved away from storing encryption key in database to ENV (#39)

Rahul Johnyb8be52176cce
diff --git a/README.md b/README.md
index 565e84400..bf0d32ff9 100644
--- a/README.md
+++ b/README.md
@@ -176,6 +176,14 @@ We just need to override default Traefik proxy routing to allow this, that's all
Run the services.
+#### Generate Encryption key to be used in backend and Platform service
+
+ Generate Fernet Key Refer https://pypi.org/project/cryptography/
+
+ `ENCRYPTION_KEY=$(python -c "import secrets, base64; print(base64.urlsafe_b64encode(secrets.token_bytes(32)).decode())")`
+
+ use the above generated encryption, key in ENV's of platform and backend
+
#### Conflicting Host Names
When same host name environment variables are used by both the service running locally and a service
diff --git a/backend/account/migrations/0005_encryptionsecret.py b/backend/account/migrations/0005_encryptionsecret.py
index d8ff25b8a..724c12522 100644
--- a/backend/account/migrations/0005_encryptionsecret.py
+++ b/backend/account/migrations/0005_encryptionsecret.py
@@ -1,9 +1,5 @@
# Generated by Django 4.2.1 on 2024-02-13 11:52
-from typing import Any
-
-from account.models import EncryptionSecret
-from cryptography.fernet import Fernet
from django.db import migrations, models
@@ -12,11 +8,6 @@ class Migration(migrations.Migration):
("account", "0004_alter_platformkey_key_name_and_more"),
]
- def initialize_secret(apps: Any, schema_editor: Any) -> None:
- EncryptionSecret.objects.create(
- key=Fernet.generate_key().decode("utf-8")
- )
-
operations = [
migrations.CreateModel(
name="EncryptionSecret",
@@ -33,7 +24,4 @@ class Migration(migrations.Migration):
("key", models.CharField(blank=True, max_length=64)),
],
),
- migrations.RunPython(
- initialize_secret, reverse_code=migrations.RunPython.noop
- ),
]
diff --git a/backend/account/migrations/0006_delete_encryptionsecret.py b/backend/account/migrations/0006_delete_encryptionsecret.py
new file mode 100644
index 000000000..1216373ee
--- /dev/null
+++ b/backend/account/migrations/0006_delete_encryptionsecret.py
@@ -0,0 +1,15 @@
+# Generated by Django 4.2.1 on 2024-03-04 05:06
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("account", "0005_encryptionsecret"),
+ ]
+
+ operations = [
+ migrations.DeleteModel(
+ name="EncryptionSecret",
+ ),
+ ]
diff --git a/backend/account/models.py b/backend/account/models.py
index c3bafc8fb..63dade30d 100644
--- a/backend/account/models.py
+++ b/backend/account/models.py
@@ -1,10 +1,11 @@
import uuid
-from backend.constants import FieldLengthConstants as FieldLength
from django.contrib.auth.models import AbstractUser, Group, Permission
from django.db import models
from django_tenants.models import DomainMixin, TenantMixin
+from backend.constants import FieldLengthConstants as FieldLength
+
NAME_SIZE = 64
KEY_SIZE = 64
@@ -131,11 +132,3 @@ class PlatformKey(models.Model):
name="unique_key_name",
),
]
-
-
-class EncryptionSecret(models.Model):
- key = models.CharField(
- max_length=KEY_SIZE,
- null=False,
- blank=True,
- )
diff --git a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py
index 5f65c0040..d5a9bb4cf 100644
--- a/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py
+++ b/backend/adapter_processor/migrations/0003_adapterinstance_adapter_metadata_b.py
@@ -3,21 +3,20 @@
import json
from typing import Any
-from account.models import EncryptionSecret
from adapter_processor.models import AdapterInstance
from cryptography.fernet import Fernet
+from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("adapter_processor", "0002_adapterinstance_unique_adapter"),
- ("account", "0005_encryptionsecret"),
]
def EncryptCredentials(apps: Any, schema_editor: Any) -> None:
- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
queryset = AdapterInstance.objects.all()
for obj in queryset: # type: ignore
diff --git a/backend/adapter_processor/serializers.py b/backend/adapter_processor/serializers.py
index bb2443450..577cd5ffb 100644
--- a/backend/adapter_processor/serializers.py
+++ b/backend/adapter_processor/serializers.py
@@ -1,10 +1,10 @@
import json
from typing import Any
-from account.models import EncryptionSecret
from adapter_processor.adapter_processor import AdapterProcessor
from adapter_processor.constants import AdapterKeys
from cryptography.fernet import Fernet
+from django.conf import settings
from rest_framework import serializers
from unstract.adapters.constants import Common as common
@@ -45,8 +45,8 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):
"""
def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]:
- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
json_string: str = json.dumps(data.pop(AdapterKeys.ADAPTER_METADATA))
data[AdapterKeys.ADAPTER_METADATA_B] = f.encrypt(
@@ -58,8 +58,8 @@ class AdapterInstanceSerializer(BaseAdapterSerializer):
def to_representation(self, instance: AdapterInstance) -> dict[str, str]:
rep: dict[str, str] = super().to_representation(instance)
- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
rep.pop(AdapterKeys.ADAPTER_METADATA_B)
adapter_metadata = json.loads(
diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py
index 4252b1483..9b9deea2f 100644
--- a/backend/backend/settings/base.py
+++ b/backend/backend/settings/base.py
@@ -143,6 +143,7 @@ API_STORAGE_DIR = os.environ.get("API_STORAGE_DIR")
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = get_required_setting("DJANGO_SECRET_KEY")
+ENCRYPTION_KEY = get_required_setting("ENCRYPTION_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
diff --git a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py
index 7f9090a2e..1e4a342c9 100644
--- a/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py
+++ b/backend/connector/migrations/0002_connectorinstance_connector_metadata_b.py
@@ -3,21 +3,20 @@
import json
from typing import Any
-from account.models import EncryptionSecret
from connector.models import ConnectorInstance
from cryptography.fernet import Fernet
+from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("connector", "0001_initial"),
- ("account", "0005_encryptionsecret"),
]
def EncryptCredentials(apps: Any, schema_editor: Any) -> None:
- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
queryset = ConnectorInstance.objects.all()
for obj in queryset: # type: ignore
diff --git a/backend/connector/models.py b/backend/connector/models.py
index bc37af041..eb0a3f7da 100644
--- a/backend/connector/models.py
+++ b/backend/connector/models.py
@@ -2,12 +2,13 @@ import json
import uuid
from typing import Any
-from account.models import EncryptionSecret, User
+from account.models import User
from connector.fields import ConnectorAuthJSONField
from connector_auth.models import ConnectorAuth
from connector_processor.connector_processor import ConnectorProcessor
from connector_processor.constants import ConnectorKeys
from cryptography.fernet import Fernet
+from django.conf import settings
from django.db import models
from project.models import Project
from utils.models.base_model import BaseModel
@@ -112,8 +113,8 @@ class ConnectorInstance(BaseModel):
@property
def metadata(self) -> Any:
- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
- cipher_suite: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ cipher_suite: Fernet = Fernet(encryption_secret.encode("utf-8"))
decrypted_value = cipher_suite.decrypt(
bytes(self.connector_metadata_b).decode("utf-8")
)
diff --git a/backend/connector/serializers.py b/backend/connector/serializers.py
index 4b664cb67..820b47d66 100644
--- a/backend/connector/serializers.py
+++ b/backend/connector/serializers.py
@@ -3,7 +3,6 @@ import logging
from collections import OrderedDict
from typing import Any, Optional
-from account.models import EncryptionSecret
from connector.constants import ConnectorInstanceKey as CIKey
from connector_auth.models import ConnectorAuth
from connector_auth.pipeline.common import ConnectorAuthHelper
@@ -11,6 +10,7 @@ from connector_processor.connector_processor import ConnectorProcessor
from connector_processor.constants import ConnectorKeys
from connector_processor.exceptions import OAuthTimeOut
from cryptography.fernet import Fernet
+from django.conf import settings
from utils.serializer_utils import SerializerUtils
from backend.serializers import AuditSerializer
@@ -55,8 +55,8 @@ class ConnectorInstanceSerializer(AuditSerializer):
)
kwargs[CIKey.CONNECTOR_MODE] = connector_mode.value
- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
json_string: str = json.dumps(kwargs.pop(CIKey.CONNECTOR_METADATA))
if self.validated_data:
self.validated_data.pop(CIKey.CONNECTOR_METADATA)
@@ -81,8 +81,8 @@ class ConnectorInstanceSerializer(AuditSerializer):
] = ConnectorProcessor.get_connector_data_with_key(
instance.connector_id, ConnectorKeys.ICON
)
- encryption_secret: EncryptionSecret = EncryptionSecret.objects.get()
- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
rep.pop(CIKey.CONNECTOR_METADATA_B)
if instance.connector_metadata_b:
diff --git a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py
index 25f4b215b..150d97371 100644
--- a/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py
+++ b/backend/prompt_studio/prompt_profile_manager/migrations/0005_removed_converter_and_added_x2text_foreign_key.py
@@ -4,15 +4,16 @@ import json
import django.db.models.deletion
from cryptography.fernet import Fernet
+from django.conf import settings
from django.db import connection, migrations, models
def fill_with_default_x2text(apps, schema):
ProfileManager = apps.get_model("prompt_profile_manager", "ProfileManager")
AdapterInstance = apps.get_model("adapter_processor", "AdapterInstance")
- EncryptionSecret = apps.get_model("account", "EncryptionSecret")
- encryption_secret = EncryptionSecret.objects.get()
- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+
+ encryption_secret: str = settings.ENCRYPTION_KEY
+ f: Fernet = Fernet(encryption_secret.encode("utf-8"))
metadata = {
"url": "http://unstract-unstructured-io:8000/general/v0/general"
}
@@ -44,7 +45,6 @@ def disable_triggers(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
- ("account", "0005_encryptionsecret"),
("adapter_processor", "0004_alter_adapterinstance_adapter_type"),
(
"prompt_profile_manager",
diff --git a/backend/sample.env b/backend/sample.env
index 1ffd6a168..5fca2ee4d 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -100,3 +100,7 @@ PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
#X2Text Service
X2TEXT_HOST=http://unstract-x2text-service
X2TEXT_PORT=3004
+
+# Encryption Key
+# key must be 32 url-safe base64-encoded bytes.
+ENCRYPTION_KEY="Sample-Key"
diff --git a/platform-service/sample.env b/platform-service/sample.env
index 03b2a1051..3d8a69a13 100644
--- a/platform-service/sample.env
+++ b/platform-service/sample.env
@@ -23,3 +23,7 @@ PG_V_PORT=5432
PG_V_USERNAME=unstract_dev
PG_V_PASSWORD=unstract_pass
PG_V_DATABASE=unstract_db
+
+# Encryption Key
+# key must be 32 url-safe base64-encoded bytes.
+ENCRYPTION_KEY="Sample-Key"
diff --git a/platform-service/src/unstract/platform_service/main.py b/platform-service/src/unstract/platform_service/main.py
index 8aad19928..627343055 100644
--- a/platform-service/src/unstract/platform_service/main.py
+++ b/platform-service/src/unstract/platform_service/main.py
@@ -45,6 +45,7 @@ PG_V_PORT = int(os.environ.get("PG_V_PORT", 5432))
PG_V_USERNAME = os.environ.get("PG_V_USERNAME", "user")
PG_V_PASSWORD = os.environ.get("PG_V_PASSWORD", "")
PG_V_DATABASE = os.environ.get("PG_V_DATABASE", "")
+ENCRYPTION_KEY = os.environ.get("ENCRYPTION_KEY")
if not (REDIS_HOST and REDIS_PORT):
raise ValueError(
"REDIS_HOST and REDIS_PORT must be set in the environment."
@@ -74,19 +75,6 @@ be_db.init(PG_BE_DATABASE)
be_db.connect()
-class EncryptionSecret(peewee.Model):
- id = peewee.IntegerField()
- key = peewee.CharField()
-
- class Meta:
- table_name = "account_encryptionsecret"
- database = be_db # This model uses the "BE_DB" database.
-
- def save(self, force_insert: bool = False, only: Any = None) -> Any:
- self.modified_at = datetime.datetime.now()
- return super().save(force_insert=force_insert, only=only)
-
-
class UnstractUsage(peewee.Model):
id = peewee.UUIDField(primary_key=True, default=uuid.uuid4)
created_at = peewee.DateTimeField(default=datetime.datetime.now)
@@ -408,8 +396,7 @@ def adapter_instance() -> Any:
)
)
- encryption_secret: EncryptionSecret = EncryptionSecret.get()
- f: Fernet = Fernet(encryption_secret.key.encode("utf-8"))
+ f: Fernet = Fernet(ENCRYPTION_KEY.encode("utf-8"))
data_dict["adapter_metadata"] = json.loads(
f.decrypt(