LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
pipileine v2 issue fix (#772)
vishnuszipstackdf1603595c5e
diff --git a/backend/pipeline_v2/migrations/0002_remove_pipeline_unique_pipeline_and_more.py b/backend/pipeline_v2/migrations/0002_remove_pipeline_unique_pipeline_and_more.pynew file mode 100644index 000000000..27e27cc2d--- /dev/null+++ b/backend/pipeline_v2/migrations/0002_remove_pipeline_unique_pipeline_and_more.py@@ -0,0 +1,69 @@+# Generated by Django 4.2.1 on 2024-10-04 10:36++import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models+++class Migration(migrations.Migration):++ dependencies = [+ ("workflow_v2", "0001_initial"),+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ("pipeline_v2", "0001_initial"),+ ]++ operations = [+ migrations.RemoveConstraint(+ model_name="pipeline",+ name="unique_pipeline",+ ),+ migrations.AlterField(+ model_name="pipeline",+ name="created_by",+ field=models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="pipelines_created",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ migrations.AlterField(+ model_name="pipeline",+ name="modified_by",+ field=models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.SET_NULL,+ related_name="pipelines_modified",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ migrations.AlterField(+ model_name="pipeline",+ name="pipeline_name",+ field=models.CharField(default="", max_length=32),+ ),+ migrations.AlterField(+ model_name="pipeline",+ name="workflow",+ field=models.ForeignKey(+ on_delete=django.db.models.deletion.CASCADE,+ related_name="pipelines",+ to="workflow_v2.workflow",+ ),+ ),+ migrations.AddConstraint(+ model_name="pipeline",+ constraint=models.UniqueConstraint(+ fields=("id", "pipeline_type"), name="unique_pipeline_entity"+ ),+ ),+ migrations.AddConstraint(+ model_name="pipeline",+ constraint=models.UniqueConstraint(+ fields=("pipeline_name", "organization"), name="unique_pipeline_name"+ ),+ ),+ ]diff --git a/backend/pipeline_v2/models.py b/backend/pipeline_v2/models.pyindex 856150439..2aeccebf4 100644--- a/backend/pipeline_v2/models.py+++ b/backend/pipeline_v2/models.py@@ -2,12 +2,13 @@ import uuidfrom account_v2.models import Userfrom django.conf import settings-from django.db import connection, models+from django.db import modelsfrom utils.models.base_model import BaseModelfrom utils.models.organization_mixin import (DefaultOrganizationManagerMixin,DefaultOrganizationMixin,)+from utils.user_context import UserContextfrom workflow_manager.workflow_v2.models.workflow import Workflowfrom backend.constants import FieldLengthConstants as FieldLength@@ -38,13 +39,11 @@ class Pipeline(DefaultOrganizationMixin, BaseModel):PAUSED = "PAUSED", "Paused"id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)- pipeline_name = models.CharField(- max_length=PIPELINE_NAME_LENGTH, default="", unique=True- )+ pipeline_name = models.CharField(max_length=PIPELINE_NAME_LENGTH, default="")workflow = models.ForeignKey(Workflow,on_delete=models.CASCADE,- related_name="pipeline_workflows",+ related_name="pipelines",null=False,blank=False,)@@ -81,27 +80,30 @@ class Pipeline(DefaultOrganizationMixin, BaseModel):created_by = models.ForeignKey(User,on_delete=models.SET_NULL,- related_name="created_pipeline",+ related_name="pipelines_created",null=True,blank=True,)modified_by = models.ForeignKey(User,on_delete=models.SET_NULL,- related_name="modified_pipeline",+ related_name="pipelines_modified",null=True,blank=True,)+ # Manager+ objects = PipelineModelManager()+@propertydef api_key_data(self):return {"pipeline": self.id, "description": f"API Key for {self.pipeline_name}"}@propertydef api_endpoint(self):- org_schema = connection.get_tenant().schema_name+ organization_id = UserContext.get_organization_identifier()deployment_endpoint = settings.API_DEPLOYMENT_PATH_PREFIX + "/pipeline/api"- api_endpoint = f"{deployment_endpoint}/{org_schema}/{self.id}/"+ api_endpoint = f"{deployment_endpoint}/{organization_id}/{self.id}/"return api_endpointdef __str__(self) -> str:@@ -120,7 +122,11 @@ class Pipeline(DefaultOrganizationMixin, BaseModel):constraints = [models.UniqueConstraint(fields=["id", "pipeline_type"],- name="unique_pipeline",+ name="unique_pipeline_entity",+ ),+ models.UniqueConstraint(+ fields=["pipeline_name", "organization"],+ name="unique_pipeline_name",),]