LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
# Generated by Django 4.2.30 on 2026-07-15 12:31
import django.utils.timezone
from django.db import migrations, models
# Seed function ported from the squashed 0005_pgorchestratorlock: leader-election
# is a plain `UPDATE` keyed on `id = 1` (with a lease predicate) and has NO
# upsert/INSERT fallback, so the single lock row must pre-exist or every
# try_acquire returns False forever. get_or_create keeps it idempotent.
def seed_single_row(apps, schema_editor):
lock_model = apps.get_model("pg_queue", "PgOrchestratorLock")
lock_model.objects.get_or_create(id=1, defaults={"leader": ""})
def unseed_single_row(apps, schema_editor):
lock_model = apps.get_model("pg_queue", "PgOrchestratorLock")
lock_model.objects.filter(id=1).delete()
class Migration(migrations.Migration):
replaces = [
("pg_queue", "0001_initial"),
("pg_queue", "0002_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more"),
("pg_queue", "0003_pgqueuemessage_pg_queue_message_priority_range"),
("pg_queue", "0004_pgbarrierstate_and_more"),
("pg_queue", "0005_pgorchestratorlock"),
("pg_queue", "0006_pgbatchdedup_and_more"),
("pg_queue", "0007_pgbarrierstate_organization_id"),
("pg_queue", "0008_pgperiodicschedule"),
(
"pg_queue",
"0009_remove_pgperiodicschedule_pg_periodic_schedule_due_idx_and_more",
),
("pg_queue", "0010_pgtaskresult"),
("pg_queue", "0011_pgbarrierstate_last_progress_at"),
("pg_queue", "0012_pgorchestrationclaim"),
("pg_queue", "0013_pgorchestrationclaim_organization_id"),
("pg_queue", "0014_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more"),
]
initial = True
dependencies = []
operations = [
migrations.CreateModel(
name="PgQueueMessage",
fields=[
("msg_id", models.BigAutoField(primary_key=True, serialize=False)),
("queue_name", models.TextField()),
("message", models.JSONField()),
("org_id", models.TextField(blank=True, default="")),
("enqueued_at", models.DateTimeField(default=django.utils.timezone.now)),
("vt", models.DateTimeField(default=django.utils.timezone.now)),
("read_ct", models.IntegerField(default=0)),
("priority", models.SmallIntegerField(default=5)),
],
options={
"db_table": "pg_queue_message",
},
),
migrations.AddConstraint(
model_name="pgqueuemessage",
constraint=models.CheckConstraint(
check=models.Q(("priority__gte", 1), ("priority__lte", 10)),
name="pg_queue_message_priority_range",
),
),
migrations.CreateModel(
name="PgBarrierState",
fields=[
("execution_id", models.TextField(primary_key=True, serialize=False)),
("remaining", models.IntegerField()),
("results", models.JSONField(default=list)),
("created_at", models.DateTimeField(default=django.utils.timezone.now)),
("expires_at", models.DateTimeField()),
],
options={
"db_table": "pg_barrier_state",
"indexes": [
models.Index(fields=["expires_at"], name="pg_barrier_expires_idx")
],
},
),
migrations.AddConstraint(
model_name="pgbarrierstate",
constraint=models.CheckConstraint(
check=models.Q(("expires_at__gt", models.F("created_at"))),
name="pg_barrier_expires_after_created",
),
),
migrations.CreateModel(
name="PgOrchestratorLock",
fields=[
("id", models.IntegerField(default=1, primary_key=True, serialize=False)),
("leader", models.TextField(blank=True, default="")),
("acquired_at", models.DateTimeField(default=django.utils.timezone.now)),
],
options={
"db_table": "pg_orchestrator_lock",
},
),
migrations.AddConstraint(
model_name="pgorchestratorlock",
constraint=models.CheckConstraint(
check=models.Q(("id", 1)), name="pg_orchestrator_lock_single_row"
),
),
migrations.RunPython(
code=seed_single_row,
reverse_code=unseed_single_row,
),
migrations.CreateModel(
name="PgBatchDedup",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("execution_id", models.TextField()),
("batch_index", models.IntegerField()),
("created_at", models.DateTimeField(default=django.utils.timezone.now)),
],
options={
"db_table": "pg_batch_dedup",
},
),
migrations.AddConstraint(
model_name="pgbatchdedup",
constraint=models.UniqueConstraint(
fields=("execution_id", "batch_index"),
name="pg_batch_dedup_exec_batch_uniq",
),
),
migrations.AddConstraint(
model_name="pgbatchdedup",
constraint=models.CheckConstraint(
check=models.Q(("batch_index__gte", 0)),
name="pg_batch_dedup_batch_index_non_negative",
),
),
migrations.AddField(
model_name="pgbarrierstate",
name="organization_id",
field=models.TextField(blank=True, default=""),
),
migrations.CreateModel(
name="PgPeriodicSchedule",
fields=[
("pipeline_id", models.UUIDField(primary_key=True, serialize=False)),
("organization_id", models.TextField(blank=True, default="")),
("workflow_id", models.UUIDField(blank=True, null=True)),
("pipeline_name", models.TextField(blank=True, default="")),
("cron_string", models.TextField()),
("enabled", models.BooleanField(default=True)),
("last_run_at", models.DateTimeField(blank=True, null=True)),
("next_run_at", models.DateTimeField(blank=True, null=True)),
("created_at", models.DateTimeField(default=django.utils.timezone.now)),
("updated_at", models.DateTimeField(auto_now=True)),
("pg_owned", models.BooleanField(default=False)),
],
options={
"db_table": "pg_periodic_schedule",
"indexes": [
models.Index(
fields=["pg_owned", "enabled", "next_run_at"],
name="pg_periodic_schedule_due_idx",
)
],
},
),
migrations.CreateModel(
name="PgTaskResult",
fields=[
("task_id", models.TextField(primary_key=True, serialize=False)),
("status", models.TextField()),
("result", models.JSONField(blank=True, null=True)),
("error", models.TextField(blank=True, default="")),
("created_at", models.DateTimeField(default=django.utils.timezone.now)),
("expires_at", models.DateTimeField(blank=True, null=True)),
],
options={
"db_table": "pg_task_result",
"indexes": [
models.Index(fields=["expires_at"], name="pg_task_result_expires_idx")
],
},
),
migrations.AddField(
model_name="pgbarrierstate",
name="last_progress_at",
field=models.DateTimeField(default=django.utils.timezone.now),
),
migrations.CreateModel(
name="PgOrchestrationClaim",
fields=[
("execution_id", models.TextField(primary_key=True, serialize=False)),
("claimed_at", models.DateTimeField(default=django.utils.timezone.now)),
("organization_id", models.TextField(blank=True, default="")),
],
options={
"db_table": "pg_orchestration_claim",
},
),
migrations.AddField(
model_name="pgqueuemessage",
name="state",
field=models.TextField(default="ready"),
),
# Django's AddField backfills every existing row to 'ready' then DROPs the
# DB default (it manages defaults at the ORM layer). Re-add a DB-level
# default so the schema is self-protecting: a raw INSERT that omits `state`
# still lands a valid 'ready' row rather than a NOT-NULL violation.
# (The ORM always sends state='ready' via the field default, so this guards
# only raw SQL; insert_message_sql() also sets it explicitly — belt +
# suspenders.) Metadata-only; touches no row data.
migrations.RunSQL(
sql="ALTER TABLE pg_queue_message ALTER COLUMN state SET DEFAULT 'ready'",
reverse_sql="ALTER TABLE pg_queue_message ALTER COLUMN state DROP DEFAULT",
),
# Deploy-safety backfill: the new claim keys off `state` and ignores `vt`,
# but AddField set EVERY pre-existing row (including claimed-but-unacked
# in-flight rows, which carry a future `vt`) to 'ready'. Without this, a
# rolling deploy over a non-empty queue would make a still-processing
# message instantly re-claimable → one-time double processing (absorbed by
# the idempotency stack, but avoidable). Land genuinely in-flight rows as
# 'claimed' so they stay invisible until their lease expires. No-op at the
# first enablement (the queue table is empty while the flag is off).
migrations.RunSQL(
sql="UPDATE pg_queue_message SET state = 'claimed' WHERE vt > now()",
reverse_sql=migrations.RunSQL.noop,
),
migrations.AddIndex(
model_name="pgqueuemessage",
index=models.Index(
models.F("queue_name"),
models.OrderBy(models.F("priority"), descending=True),
models.F("msg_id"),
condition=models.Q(("state", "ready")),
name="pg_queue_message_claim_idx",
),
),
migrations.AddIndex(
model_name="pgqueuemessage",
index=models.Index(
models.F("msg_id"),
condition=models.Q(("state", "claimed")),
name="pg_queue_message_claimed_idx",
),
),
migrations.AddConstraint(
model_name="pgqueuemessage",
constraint=models.CheckConstraint(
check=models.Q(("state__in", ["ready", "claimed"])),
name="pg_queue_message_state_valid",
),
),
# NOTE: performance tuning for the state-machine churn
# (fillfactor + aggressive per-table autovacuum) is deliberately KEPT OUT of
# this migration. Those are core-Postgres reloptions, but they are (a)
# environment-specific tuning that belongs in the ops/deploy layer, not app
# schema, and (b) potentially restricted by a locked-down managed role
# (Cloud SQL / RDS / Azure), where a failed `ALTER TABLE SET (autovacuum_*)`
# would abort the whole migration. The fix itself (state column + partial
# indexes) is pure portable DDL and needs none of it. If a soak shows bloat,
# apply the tuning per environment via ops (see the PG-Queue Claim State
# runbook in Confluence), not here.
]