Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

11.8 KB
Raw Blame
# Generated by Django 4.2.30 on 2026-07-15 12:31

import django.utils.timezone
from django.db import migrations, models


# Seed function ported from the squashed 0005_pgorchestratorlock: leader-election
# is a plain `UPDATE` keyed on `id = 1` (with a lease predicate) and has NO
# upsert/INSERT fallback, so the single lock row must pre-exist or every
# try_acquire returns False forever. get_or_create keeps it idempotent.
def seed_single_row(apps, schema_editor):
    lock_model = apps.get_model("pg_queue", "PgOrchestratorLock")
    lock_model.objects.get_or_create(id=1, defaults={"leader": ""})


def unseed_single_row(apps, schema_editor):
    lock_model = apps.get_model("pg_queue", "PgOrchestratorLock")
    lock_model.objects.filter(id=1).delete()


class Migration(migrations.Migration):
    replaces = [
        ("pg_queue", "0001_initial"),
        ("pg_queue", "0002_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more"),
        ("pg_queue", "0003_pgqueuemessage_pg_queue_message_priority_range"),
        ("pg_queue", "0004_pgbarrierstate_and_more"),
        ("pg_queue", "0005_pgorchestratorlock"),
        ("pg_queue", "0006_pgbatchdedup_and_more"),
        ("pg_queue", "0007_pgbarrierstate_organization_id"),
        ("pg_queue", "0008_pgperiodicschedule"),
        (
            "pg_queue",
            "0009_remove_pgperiodicschedule_pg_periodic_schedule_due_idx_and_more",
        ),
        ("pg_queue", "0010_pgtaskresult"),
        ("pg_queue", "0011_pgbarrierstate_last_progress_at"),
        ("pg_queue", "0012_pgorchestrationclaim"),
        ("pg_queue", "0013_pgorchestrationclaim_organization_id"),
        ("pg_queue", "0014_remove_pgqueuemessage_pg_queue_message_dequeue_idx_and_more"),
    ]

    initial = True

    dependencies = []

    operations = [
        migrations.CreateModel(
            name="PgQueueMessage",
            fields=[
                ("msg_id", models.BigAutoField(primary_key=True, serialize=False)),
                ("queue_name", models.TextField()),
                ("message", models.JSONField()),
                ("org_id", models.TextField(blank=True, default="")),
                ("enqueued_at", models.DateTimeField(default=django.utils.timezone.now)),
                ("vt", models.DateTimeField(default=django.utils.timezone.now)),
                ("read_ct", models.IntegerField(default=0)),
                ("priority", models.SmallIntegerField(default=5)),
            ],
            options={
                "db_table": "pg_queue_message",
            },
        ),
        migrations.AddConstraint(
            model_name="pgqueuemessage",
            constraint=models.CheckConstraint(
                check=models.Q(("priority__gte", 1), ("priority__lte", 10)),
                name="pg_queue_message_priority_range",
            ),
        ),
        migrations.CreateModel(
            name="PgBarrierState",
            fields=[
                ("execution_id", models.TextField(primary_key=True, serialize=False)),
                ("remaining", models.IntegerField()),
                ("results", models.JSONField(default=list)),
                ("created_at", models.DateTimeField(default=django.utils.timezone.now)),
                ("expires_at", models.DateTimeField()),
            ],
            options={
                "db_table": "pg_barrier_state",
                "indexes": [
                    models.Index(fields=["expires_at"], name="pg_barrier_expires_idx")
                ],
            },
        ),
        migrations.AddConstraint(
            model_name="pgbarrierstate",
            constraint=models.CheckConstraint(
                check=models.Q(("expires_at__gt", models.F("created_at"))),
                name="pg_barrier_expires_after_created",
            ),
        ),
        migrations.CreateModel(
            name="PgOrchestratorLock",
            fields=[
                ("id", models.IntegerField(default=1, primary_key=True, serialize=False)),
                ("leader", models.TextField(blank=True, default="")),
                ("acquired_at", models.DateTimeField(default=django.utils.timezone.now)),
            ],
            options={
                "db_table": "pg_orchestrator_lock",
            },
        ),
        migrations.AddConstraint(
            model_name="pgorchestratorlock",
            constraint=models.CheckConstraint(
                check=models.Q(("id", 1)), name="pg_orchestrator_lock_single_row"
            ),
        ),
        migrations.RunPython(
            code=seed_single_row,
            reverse_code=unseed_single_row,
        ),
        migrations.CreateModel(
            name="PgBatchDedup",
            fields=[
                (
                    "id",
                    models.BigAutoField(
                        auto_created=True,
                        primary_key=True,
                        serialize=False,
                        verbose_name="ID",
                    ),
                ),
                ("execution_id", models.TextField()),
                ("batch_index", models.IntegerField()),
                ("created_at", models.DateTimeField(default=django.utils.timezone.now)),
            ],
            options={
                "db_table": "pg_batch_dedup",
            },
        ),
        migrations.AddConstraint(
            model_name="pgbatchdedup",
            constraint=models.UniqueConstraint(
                fields=("execution_id", "batch_index"),
                name="pg_batch_dedup_exec_batch_uniq",
            ),
        ),
        migrations.AddConstraint(
            model_name="pgbatchdedup",
            constraint=models.CheckConstraint(
                check=models.Q(("batch_index__gte", 0)),
                name="pg_batch_dedup_batch_index_non_negative",
            ),
        ),
        migrations.AddField(
            model_name="pgbarrierstate",
            name="organization_id",
            field=models.TextField(blank=True, default=""),
        ),
        migrations.CreateModel(
            name="PgPeriodicSchedule",
            fields=[
                ("pipeline_id", models.UUIDField(primary_key=True, serialize=False)),
                ("organization_id", models.TextField(blank=True, default="")),
                ("workflow_id", models.UUIDField(blank=True, null=True)),
                ("pipeline_name", models.TextField(blank=True, default="")),
                ("cron_string", models.TextField()),
                ("enabled", models.BooleanField(default=True)),
                ("last_run_at", models.DateTimeField(blank=True, null=True)),
                ("next_run_at", models.DateTimeField(blank=True, null=True)),
                ("created_at", models.DateTimeField(default=django.utils.timezone.now)),
                ("updated_at", models.DateTimeField(auto_now=True)),
                ("pg_owned", models.BooleanField(default=False)),
            ],
            options={
                "db_table": "pg_periodic_schedule",
                "indexes": [
                    models.Index(
                        fields=["pg_owned", "enabled", "next_run_at"],
                        name="pg_periodic_schedule_due_idx",
                    )
                ],
            },
        ),
        migrations.CreateModel(
            name="PgTaskResult",
            fields=[
                ("task_id", models.TextField(primary_key=True, serialize=False)),
                ("status", models.TextField()),
                ("result", models.JSONField(blank=True, null=True)),
                ("error", models.TextField(blank=True, default="")),
                ("created_at", models.DateTimeField(default=django.utils.timezone.now)),
                ("expires_at", models.DateTimeField(blank=True, null=True)),
            ],
            options={
                "db_table": "pg_task_result",
                "indexes": [
                    models.Index(fields=["expires_at"], name="pg_task_result_expires_idx")
                ],
            },
        ),
        migrations.AddField(
            model_name="pgbarrierstate",
            name="last_progress_at",
            field=models.DateTimeField(default=django.utils.timezone.now),
        ),
        migrations.CreateModel(
            name="PgOrchestrationClaim",
            fields=[
                ("execution_id", models.TextField(primary_key=True, serialize=False)),
                ("claimed_at", models.DateTimeField(default=django.utils.timezone.now)),
                ("organization_id", models.TextField(blank=True, default="")),
            ],
            options={
                "db_table": "pg_orchestration_claim",
            },
        ),
        migrations.AddField(
            model_name="pgqueuemessage",
            name="state",
            field=models.TextField(default="ready"),
        ),
        # Django's AddField backfills every existing row to 'ready' then DROPs the
        # DB default (it manages defaults at the ORM layer). Re-add a DB-level
        # default so the schema is self-protecting: a raw INSERT that omits `state`
        # still lands a valid 'ready' row rather than a NOT-NULL violation.
        # (The ORM always sends state='ready' via the field default, so this guards
        # only raw SQL; insert_message_sql() also sets it explicitly — belt +
        # suspenders.) Metadata-only; touches no row data.
        migrations.RunSQL(
            sql="ALTER TABLE pg_queue_message ALTER COLUMN state SET DEFAULT 'ready'",
            reverse_sql="ALTER TABLE pg_queue_message ALTER COLUMN state DROP DEFAULT",
        ),
        # Deploy-safety backfill: the new claim keys off `state` and ignores `vt`,
        # but AddField set EVERY pre-existing row (including claimed-but-unacked
        # in-flight rows, which carry a future `vt`) to 'ready'. Without this, a
        # rolling deploy over a non-empty queue would make a still-processing
        # message instantly re-claimable → one-time double processing (absorbed by
        # the idempotency stack, but avoidable). Land genuinely in-flight rows as
        # 'claimed' so they stay invisible until their lease expires. No-op at the
        # first enablement (the queue table is empty while the flag is off).
        migrations.RunSQL(
            sql="UPDATE pg_queue_message SET state = 'claimed' WHERE vt > now()",
            reverse_sql=migrations.RunSQL.noop,
        ),
        migrations.AddIndex(
            model_name="pgqueuemessage",
            index=models.Index(
                models.F("queue_name"),
                models.OrderBy(models.F("priority"), descending=True),
                models.F("msg_id"),
                condition=models.Q(("state", "ready")),
                name="pg_queue_message_claim_idx",
            ),
        ),
        migrations.AddIndex(
            model_name="pgqueuemessage",
            index=models.Index(
                models.F("msg_id"),
                condition=models.Q(("state", "claimed")),
                name="pg_queue_message_claimed_idx",
            ),
        ),
        migrations.AddConstraint(
            model_name="pgqueuemessage",
            constraint=models.CheckConstraint(
                check=models.Q(("state__in", ["ready", "claimed"])),
                name="pg_queue_message_state_valid",
            ),
        ),
        # NOTE: performance tuning for the state-machine churn
        # (fillfactor + aggressive per-table autovacuum) is deliberately KEPT OUT of
        # this migration. Those are core-Postgres reloptions, but they are (a)
        # environment-specific tuning that belongs in the ops/deploy layer, not app
        # schema, and (b) potentially restricted by a locked-down managed role
        # (Cloud SQL / RDS / Azure), where a failed `ALTER TABLE SET (autovacuum_*)`
        # would abort the whole migration. The fix itself (state column + partial
        # indexes) is pure portable DDL and needs none of it. If a soak shows bloat,
        # apply the tuning per environment via ops (see the PG-Queue Claim State
        # runbook in Confluence), not here.
    ]