LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import logging
from typing import Any
from pipeline_v2.manager import PipelineManager
from rest_framework import serializers
from backend.constants import FieldLengthConstants as FieldLength
from scheduler.constants import SchedulerConstants as SC
logger = logging.getLogger(__name__)
JOB_NAME_LENGTH = 255
class JobKwargsSerializer(serializers.Serializer):
verb = serializers.CharField(max_length=6)
# TODO: Add custom URL field to allow URLs for running in docker
# url = serializers.URLField()
url = serializers.CharField(max_length=128)
headers = serializers.JSONField()
data = serializers.JSONField()
class AddJobSerializer(serializers.Serializer):
id = serializers.CharField(max_length=FieldLength.UUID_LENGTH)
cron_string = serializers.CharField(
max_length=FieldLength.CRON_LENGTH,
allow_null=True,
required=False,
allow_blank=True,
)
name = serializers.CharField(
max_length=JOB_NAME_LENGTH, required=False, allow_blank=True
)
job_kwargs = JobKwargsSerializer(write_only=True)
def to_internal_value(self, data: dict[str, Any]) -> dict[str, Any]:
if SC.NAME not in data:
data[SC.NAME] = f"Job-{data[SC.ID]}"
data[SC.JOB_KWARGS] = (
PipelineManager.get_pipeline_execution_data_for_scheduled_run(
pipeline_id=data[SC.ID]
)
)
return super().to_internal_value(data) # type: ignore