:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 3 missing tables to the Unstructured source (#119732)
Tom Owers3877eabcdd33
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex a4c7aea359d6..639913b96f4f 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -1166,8 +1166,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/uk_companies_house/uk_companies_house.py1 products/warehouse_sources/backend/temporal/data_imports/sources/unleash/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/unleash/unleash.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py1 products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/uppromote.py1 products/warehouse_sources/backend/temporal/data_imports/sources/upstash/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/uptimerobot/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 70630ca942c4..82008cf78a39 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -12762,9 +12762,9 @@ Today (4): `destinations`, `jobs`, `sources`, `workflows`Diffed against: <https://docs.unstructured.io/llms.txt>-- [ ] `job failed files (GET /jobs/{id}/failed-files)` — per-file failure reasons — the only place processing errors are enumerated; jobs alone give just a status (high)-- [ ] `job details (GET /jobs/{id}/details)` — per-node processing counts and timings inside a job, the grain needed for throughput and cost analysis (high)-- [ ] `notifications (GET /notifications)` — workspace event stream (job started/failed/completed) with timestamps, an analytical log the jobs table does not reproduce (medium)+- [x] `job failed files (GET /jobs/{id}/failed-files)` — per-file failure reasons — the only place processing errors are enumerated; jobs alone give just a status (high)+- [x] `job details (GET /jobs/{id}/details)` — per-node processing counts and timings inside a job, the grain needed for throughput and cost analysis (high)+- [x] `notifications (GET /notifications)` — workspace event stream (job started/failed/completed) with timestamps, an analytical log the jobs table does not reproduce (medium)- [ ] `templates (GET /templates)` — lookup resolving the template a workflow was created from, plus its DAG metadata (medium)- [ ] `source/destination connection checks (GET /sources/{id}/connection-check, /destinations/{id}/connection-check)` — connector health history to correlate with failed jobs (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/canonical_descriptions.pyindex 3b13a7997ebc..d472fe43da2e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/canonical_descriptions.py@@ -64,4 +64,45 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"key": "Optional stable key for the destination connector.",},},+ "job_details": {+ "description": "Processing details of each job: per-node document counts, overall outcome, and progress.",+ "docs_url": "https://docs.unstructured.io/api-reference/workflow/jobs",+ "columns": {+ "id": "Unique identifier of the job.",+ "workflow_id": "Identifier of the workflow this job ran.",+ "job_created_at": "Timestamp when the job was created.",+ "processing_status": "Document-level outcome of the job (e.g. SUCCESS, COMPLETED_WITH_ERRORS, FAILED).",+ "node_stats": "Per-node document counts (ready, in progress, success, failure) for each step of the workflow DAG.",+ "summary": "Totals of documents processed, succeeded, and failed across the job.",+ "message": "Human-readable message about the job's processing state.",+ "progress": "Advisory reading of why the job is not finished yet, including when its steps last reported.",+ "queued_behind_job_ids_snapshot": "Jobs that held the workspace's concurrency slots when this job was queued.",+ "completion_issue": "Why a job completed without delivering a document, if an issue was detected.",+ },+ },+ "job_failed_files": {+ "description": "Files that failed to process in a job, with the workflow node that failed each one and the error.",+ "docs_url": "https://docs.unstructured.io/api-reference/workflow/jobs",+ "columns": {+ "job_id": "Identifier of the job the file failed in.",+ "workflow_id": "Identifier of the workflow the job ran.",+ "job_created_at": "Timestamp when the job was created.",+ "document": "The file that failed to process.",+ "error": "Error message for the failure.",+ "node": "The workflow node that failed the file (id, name, type, and subtype).",+ "node_id": "Identifier of the workflow node that failed the file.",+ },+ },+ "notifications": {+ "description": "Platform notification events, such as a job that started, completed, or failed.",+ "docs_url": "https://docs.unstructured.io/api-reference",+ "columns": {+ "id": "Unique identifier of the notification event.",+ "event_type": "Event type (e.g. job.completed, job.failed).",+ "workflow_id": "Workflow the event relates to; null for account-level events.",+ "payload": "Event payload data.",+ "created_at": "Timestamp when the event was created.",+ "is_read": "Whether the user that owns the API key marked the notification as read.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/posthog_com_doc.mdx b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/posthog_com_doc.mdxindex 674c80c580fd..cd6da2311a30 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/posthog_com_doc.mdx+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/posthog_com_doc.mdx@@ -22,7 +22,7 @@ import AlphaRelease from '../_snippets/alpha-release.mdx'<AlphaRelease />-[Unstructured](https://unstructured.io) is a document ETL platform that turns raw files (PDFs, Office documents, emails, and more) into RAG-ready structured data. This source pulls the metadata from your Unstructured platform account, its workflows, job runs, and source/destination connectors, into the PostHog data warehouse so you can monitor the health of your document pipelines alongside the rest of your data.+[Unstructured](https://unstructured.io) is a document ETL platform that turns raw files (PDFs, Office documents, emails, and more) into RAG-ready structured data. This source pulls the metadata from your Unstructured platform account, its workflows, job runs, per-job processing details and failed files, notifications, and source/destination connectors, into the PostHog data warehouse so you can monitor the health of your document pipelines alongside the rest of your data.## Prerequisites@@ -45,7 +45,9 @@ Leave **API host** blank unless Unstructured provisioned your account with a cus<SyncModes />-Unstructured's read endpoints do not expose a reliable server-side incremental filter, and workflows and connectors are mutable configuration objects, so every table syncs as a full refresh. These are low-volume inventory tables, so a full refresh each sync is inexpensive and always reflects the current state.+The `notifications` table supports incremental sync on `created_at`. Every other table syncs as a full refresh: workflows and connectors are mutable configuration objects, and jobs have no server-side time filter. These are low-volume inventory tables, so a full refresh each sync always reflects the current state.++The `job_details` and `job_failed_files` tables make one request for each job on every sync. `job_failed_files` skips jobs that report no failed files. `job_details` is off by default, so turn it on only when you need per-node processing counts.## Configurationdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.pyindex fe39dc6ef566..e2f8149f872b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py@@ -1,21 +1,24 @@from dataclasses import dataclass, field+from typing import Literal-from products.warehouse_sources.backend.types import IncrementalField+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-# Unstructured's Platform API paths always carry a trailing slash; hitting them without it returns a-# 307 redirect (which also downgrades the scheme), so keep the slash to talk to the API directly.+# Paths match the Platform API spec exactly: collection paths carry a trailing slash, and the per-job+# and notification paths do not. A collection path without its slash returns a 307 redirect (which+# also downgrades the scheme), so keep each path as the spec writes it.-@dataclass+@dataclass(frozen=True)class UnstructuredEndpointConfig:name: strpath: str# Field to partition Delta files by. Must be stable (never rewritten upstream) so partitions# don't churn every sync; `created_at` is set once at object creation across every resource here.+ # The per-job tables carry no timestamp of their own, so they partition on the parent job's.partition_key: str | None = "created_at"primary_keys: list[str] = field(default_factory=lambda: ["id"])- # Only /workflows/ accepts page / page_size; the other list endpoints return the full array in a- # single response, so pagination (and resume) only applies to workflows.+ # Only /workflows/ accepts page / page_size. /notifications and the per-job failed files have their+ # own cursor and offset paging; the other list endpoints return the full array in one response.paginated: bool = Falseshould_sync_default: bool = True# Top-level fields dropped from every row before it is yielded. The /sources/ and /destinations/@@ -24,6 +27,9 @@ class UnstructuredEndpointConfig:# access could read it back. We drop the whole object rather than allowlisting because the fields# vary per connector type and a new type could introduce a secret we don't yet know to strip.drop_fields: tuple[str, ...] = ()+ # Per-job endpoints: list /jobs/ first, then call `path` (with a `{job_id}` placeholder) once per job.+ fan_out_jobs: bool = False+ sort_mode: Literal["asc", "desc"] = "asc"UNSTRUCTURED_ENDPOINTS: dict[str, UnstructuredEndpointConfig] = {@@ -33,14 +39,41 @@ UNSTRUCTURED_ENDPOINTS: dict[str, UnstructuredEndpointConfig] = {"destinations": UnstructuredEndpointConfig(name="destinations", path="/api/v1/destinations/", drop_fields=("config",)),+ # One request per job on every sync, so it is off by default to keep the request volume opt-in.+ "job_details": UnstructuredEndpointConfig(+ name="job_details",+ path="/api/v1/jobs/{job_id}/details",+ partition_key="job_created_at",+ fan_out_jobs=True,+ should_sync_default=False,+ ),+ # A job reports each failure once per file and workflow node, so neither field is unique alone.+ "job_failed_files": UnstructuredEndpointConfig(+ name="job_failed_files",+ path="/api/v1/jobs/{job_id}/failed-files",+ partition_key="job_created_at",+ primary_keys=["job_id", "node_id", "document"],+ fan_out_jobs=True,+ ),+ # The API returns notifications newest first and accepts no sort parameter.+ "notifications": UnstructuredEndpointConfig(name="notifications", path="/api/v1/notifications", sort_mode="desc"),}ENDPOINTS = tuple(UNSTRUCTURED_ENDPOINTS.keys())-# All four endpoints ship as full refresh. Only /workflows/ exposes a server-side timestamp filter-# (`created_since`); jobs, sources, and destinations have no server-side time filter at all (confirmed-# against the Platform API OpenAPI spec), so a "since" sync would still page the whole list and isn't-# genuinely incremental. Workflows, sources, and destinations are also low-volume mutable config-# inventory where a full refresh yields the correct current state (status/schedule edits included),-# which incremental-on-created_at would freeze. See the module docstring in `unstructured.py`.-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}+# Only notifications sync incrementally: they are an append-only event log with a server-side `since`+# filter on `created_at`. /workflows/ also has a `created_since` filter, but workflows, sources, and+# destinations are low-volume mutable config inventory where a full refresh yields the correct current+# state (status/schedule edits included), which incremental-on-created_at would freeze. Jobs and the+# per-job tables have no server-side time filter at all (confirmed against the Platform API OpenAPI+# spec), so a "since" sync would still page the whole list. See the module docstring in `unstructured.py`.+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ "notifications": [+ {+ "label": "created_at",+ "type": IncrementalFieldType.DateTime,+ "field": "created_at",+ "field_type": IncrementalFieldType.DateTime,+ }+ ],+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/source.pyindex 0bd42cd47c07..98985a568a5d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/source.py@@ -157,4 +157,6 @@ Leave **API host** blank unless Unstructured provisioned your account with a cuslogger=inputs.logger,resumable_source_manager=resumable_source_manager,team_id=inputs.team_id,+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/tests/test_unstructured.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/tests/test_unstructured.pyindex 4ddd4d69a8cd..e3615a222505 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/tests/test_unstructured.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/tests/test_unstructured.py@@ -1,3 +1,6 @@+from datetime import UTC, datetime+from typing import Any+from unittest.mock import patchimport requests@@ -7,6 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.unstructured import (@@ -15,6 +19,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_from products.warehouse_sources.backend.temporal.data_imports.sources.unstructured.source import UnstructuredSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.unstructured.unstructured import (DEFAULT_BASE_URL,+ FAILED_FILES_PAGE_SIZE,+ NOTIFICATIONS_PAGE_SIZE,WORKFLOWS_PAGE_SIZE,UnstructuredResumeConfig,UnstructuredRetryableError,@@ -161,6 +167,183 @@ class TestGetRows:assert result.requests == []+JOB_A = "11111111-1111-4111-8111-111111111111"+JOB_B = "22222222-2222-4222-8222-222222222222"+JOB_C = "33333333-3333-4333-8333-333333333333"+++def _job(job_id: str, failed_file_count: int | None = None) -> dict[str, Any]:+ return {+ "id": job_id,+ "workflow_id": "wf-1",+ "created_at": "2026-01-02T03:04:05Z",+ "status": "COMPLETED",+ "failed_file_count": failed_file_count,+ }+++def _failed_file(document: str, node_id: str = "node-partition") -> dict[str, Any]:+ return {"document": document, "error": "could not parse", "node": {"id": node_id, "type": "partition"}}+++class TestJobFanOut:+ def test_failed_files_skips_clean_jobs_and_walks_offset_pages(self) -> None:+ result = _driver().run(+ "job_failed_files",+ route(+ {+ "/api/v1/jobs/": [ScriptedResponse(json=[_job(JOB_A, 0), _job(JOB_B, 3), _job(JOB_C)])],+ f"/api/v1/jobs/{JOB_B}/failed-files": [+ ScriptedResponse(+ json={"failed_files": [_failed_file("a.pdf"), _failed_file("b.pdf")], "next_offset": 2}+ ),+ ScriptedResponse(json={"failed_files": [_failed_file("c.pdf")], "next_offset": None}),+ ],+ f"/api/v1/jobs/{JOB_C}/failed-files": [ScriptedResponse(status=404, json={"detail": "gone"})],+ }+ ),+ )++ assert result.raised is None+ assert f"/api/v1/jobs/{JOB_A}/failed-files" not in result.paths+ assert result.params("offset") == [None, "0", "2", "0"]+ assert result.params("limit")[1:] == [str(FAILED_FILES_PAGE_SIZE)] * 3+ assert [row["document"] for row in result.rows] == ["a.pdf", "b.pdf", "c.pdf"]+ assert result.rows[0] == {+ **_failed_file("a.pdf"),+ "job_id": JOB_B,+ "workflow_id": "wf-1",+ "job_created_at": "2026-01-02T03:04:05Z",+ "node_id": "node-partition",+ }++ @parameterized.expand(+ [+ ("no_next_offset", {"failed_files": [{"document": "a.pdf", "error": "x", "node": {"id": "n"}}]}),+ ("next_offset_not_ahead", {"failed_files": [], "next_offset": 0}),+ ]+ )+ def test_failed_files_stops_without_a_forward_offset(self, _name: str, body: dict[str, Any]) -> None:+ result = _driver().run(+ "job_failed_files",+ route(+ {+ "/api/v1/jobs/": [ScriptedResponse(json=[_job(JOB_A, 1)])],+ f"/api/v1/jobs/{JOB_A}/failed-files": [ScriptedResponse(json=body)],+ }+ ),+ )++ assert result.raised is None+ assert len(result.requests) == 2++ def test_job_details_adds_job_fields_and_skips_deleted_jobs(self) -> None:+ details = {"id": JOB_A, "processing_status": "SUCCESS", "node_stats": [{"node_id": "n", "success": 4}]}+ result = _driver().run(+ "job_details",+ route(+ {+ "/api/v1/jobs/": [ScriptedResponse(json=[_job(JOB_A), _job(JOB_B)])],+ f"/api/v1/jobs/{JOB_A}/details": [ScriptedResponse(json=details)],+ f"/api/v1/jobs/{JOB_B}/details": [ScriptedResponse(status=404, json={"detail": "gone"})],+ }+ ),+ )++ assert result.raised is None+ assert result.rows == [{**details, "workflow_id": "wf-1", "job_created_at": "2026-01-02T03:04:05Z"}]++ def test_per_job_server_error_still_fails_the_sync(self) -> None:+ # Only a 404 means the job is gone; any other error must not silently drop the job's rows.+ result = _driver().run(+ "job_details",+ route(+ {+ "/api/v1/jobs/": [ScriptedResponse(json=[_job(JOB_A)])],+ f"/api/v1/jobs/{JOB_A}/details": [ScriptedResponse(status=403, json={"detail": "no"})],+ }+ ),+ )++ assert isinstance(result.raised, requests.HTTPError)+++def _event(event_id: str, created_at: str = "2026-03-01T00:00:00Z") -> dict[str, Any]:+ return {"id": event_id, "event_type": "job.completed", "payload": {}, "created_at": created_at}+++class TestNotifications:+ def test_walks_cursor_pages_and_stages_each_cursor_before_its_page(self) -> None:+ result = _driver().run(+ "notifications",+ [+ ScriptedResponse(json={"events": [_event("e1"), _event("e2")], "next_cursor": "c1"}),+ ScriptedResponse(json={"events": [], "next_cursor": "c2"}),+ ScriptedResponse(json={"events": [_event("e3")], "next_cursor": None}),+ ],+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["e1", "e2", "e3"]+ assert result.params("cursor") == [None, "c1", "c2"]+ assert result.params("limit") == [str(NOTIFICATIONS_PAGE_SIZE)] * 3+ assert result.saved_states == [+ UnstructuredResumeConfig(next_cursor="c1"),+ UnstructuredResumeConfig(next_cursor="c2"),+ ]+ assert result.committed_states == result.saved_states+ assert result.response is not None and result.response.sort_mode == "desc"++ def test_resumes_from_saved_cursor(self) -> None:+ result = _driver().run(+ "notifications",+ [ScriptedResponse(json={"events": [_event("e9")], "next_cursor": None})],+ resume_state=UnstructuredResumeConfig(next_cursor="c7"),+ )++ assert result.params("cursor") == ["c7"]+ assert [row["id"] for row in result.rows] == ["e9"]++ def test_repeated_cursor_ends_the_walk(self) -> None:+ result = _driver().run(+ "notifications",+ [+ ScriptedResponse(json={"events": [_event("e1")], "next_cursor": "c1"}),+ ScriptedResponse(json={"events": [_event("e1")], "next_cursor": "c1"}),+ ],+ )++ assert result.raised is None+ assert len(result.requests) == 2++ @parameterized.expand(+ [+ ("incremental_datetime", True, datetime(2026, 2, 1, 12, 0), "2026-02-01T12:00:00+00:00"),+ ("incremental_string", True, "2026-02-01T12:00:00Z", "2026-02-01T12:00:00Z"),+ ("incremental_first_sync", True, None, None),+ ("full_refresh_ignores_watermark", False, datetime(2026, 2, 1, tzinfo=UTC), None),+ ]+ )+ def test_since_follows_the_incremental_watermark(+ self, _name: str, incremental: bool, last_value: Any, expected_since: str | None+ ) -> None:+ result = _driver().run(+ "notifications",+ [ScriptedResponse(json={"events": [], "next_cursor": None})],+ incremental_field="created_at",+ should_use_incremental_field=incremental,+ db_incremental_field_last_value=last_value,+ )++ assert result.params("since") == [expected_since]++ def test_non_object_body_returns_empty(self) -> None:+ result = _driver().run("notifications", [ScriptedResponse(json=[])])++ assert result.raised is None+ assert result.rows == []++class TestUnstructuredSource:@parameterized.expand(["workflows", "jobs", "sources", "destinations"])def test_source_response_contract(self, endpoint: str) -> None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.pyindex 153edd4ca9a5..a29f0db96955 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py@@ -5,22 +5,28 @@ list endpoints are synced: `/api/v1/workflows/`, `/api/v1/jobs/`, `/api/v1/sourc`/api/v1/destinations/`. Each returns a bare JSON array of objects (no envelope). Only `/workflows/`is paginated (page / page_size); the rest return their whole list in one response.-Incremental sync is intentionally not offered. Only `/workflows/` exposes a server-side timestamp-filter (`created_since`), and even there the resource is mutable config (status / schedule / updated_at-change over time), so filtering on `created_at` would freeze already-synced rows and miss updates. The-jobs / sources / destinations endpoints expose no server-side time filter at all, so a "since" sync-would re-page the full list anyway. These are all low-volume inventory tables, so every stream ships as-a full refresh, which reflects the correct current state each sync. The `created_since` filter and-per-endpoint behavior were verified against the live Platform API OpenAPI spec (version 3.1.1); they-were not smoke-tested against a real key, so a future implementer enabling incremental should confirm-`created_since` actually filters before relying on it. The workflows `sort_by=created_at` below is-similarly taken from the spec (sort_by is a free-form string defaulting to `id`) but unverified against-a live key; if the API rejects it, drop the sort params and fall back to the default `id` ordering.+Two per-job tables fan out over `/api/v1/jobs/`: `/jobs/{id}/details` (one object per job) and+`/jobs/{id}/failed-files` (offset paging with `next_offset`). `/api/v1/notifications` is an event log+wrapped in `{"events": [...], "next_cursor": ...}`, ordered newest first with cursor paging.++Incremental sync is offered only for notifications, through the server-side `since` filter. Only+`/workflows/` also exposes a server-side timestamp filter (`created_since`), and there the resource is+mutable config (status / schedule / updated_at change over time), so filtering on `created_at` would+freeze already-synced rows and miss updates. The jobs / sources / destinations endpoints expose no+server-side time filter at all, so a "since" sync would re-page the full list anyway. These are all+low-volume inventory tables, so they ship as a full refresh, which reflects the correct current state+each sync. The filters and per-endpoint behavior were verified against the live Platform API OpenAPI+spec (versions 3.1.1 and 3.3.0); they were not smoke-tested against a real key, so a future implementer+enabling more incremental tables should confirm `created_since` actually filters before relying on it.+The workflows `sort_by=created_at` below is similarly taken from the spec (sort_by is a free-form string+defaulting to `id`) but unverified against a live key; if the API rejects it, drop the sort params and+fall back to the default `id` ordering."""import reimport dataclassesfrom collections.abc import Iterator+from datetime import UTC, datetimefrom typing import Any, Optionalfrom urllib.parse import urlparse@@ -40,6 +46,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.unstructurDEFAULT_BASE_URL = "https://platform.unstructuredapp.io"# /workflows/ default page_size is 20; pull larger pages to cut round-trips on busy accounts.WORKFLOWS_PAGE_SIZE = 100+# The largest page sizes the spec allows for these endpoints.+NOTIFICATIONS_PAGE_SIZE = 100+FAILED_FILES_PAGE_SIZE = 1000+JOBS_PATH = "/api/v1/jobs/"REQUEST_TIMEOUT = 60HOST_NOT_ALLOWED_ERROR = "This API host is not allowed"@@ -48,11 +58,12 @@ class UnstructuredRetryableError(Exception):pass[email protected](frozen=True)class UnstructuredResumeConfig:- # 1-based index of the next /workflows/ page to fetch. Only workflows paginate, so this is the- # sole piece of resumable state.- next_page: int+ # 1-based index of the next /workflows/ page to fetch.+ next_page: int = 1+ # Cursor of the next /notifications page.+ next_cursor: str | None = Nonedef normalize_base_url(base_url: str | None) -> str:@@ -112,13 +123,14 @@ def _drop_sensitive_fields(rows: list[dict[str, Any]], drop_fields: tuple[str, .wait=wait_exponential_jitter(initial=1, max=30),reraise=True,)-def _fetch(+def _fetch_json(session: requests.Session,url: str,headers: dict[str, str],params: Optional[dict[str, Any]],logger: FilteringBoundLogger,-) -> list[dict[str, Any]]:+ allow_not_found: bool = False,+) -> Any:response = session.get(url, headers=headers, params=params, timeout=REQUEST_TIMEOUT)# The session never follows redirects (SSRF hardening); surface a 30x as an error rather than@@ -134,11 +146,25 @@ def _fetch(f"Unstructured API error (retryable): status={response.status_code}, url={url}")+ # A job deleted between the /jobs/ listing and its per-job request has nothing left to sync.+ if allow_not_found and response.status_code == 404:+ return None+if not response.ok:logger.error(f"Unstructured API error: status={response.status_code}, body={response.text}, url={url}")response.raise_for_status()- data = response.json()+ return response.json()+++def _fetch(+ session: requests.Session,+ url: str,+ headers: dict[str, str],+ params: Optional[dict[str, Any]],+ logger: FilteringBoundLogger,+) -> list[dict[str, Any]]:+ data = _fetch_json(session, url, headers, params, logger)# Every list endpoint returns a bare array; guard against an unexpected envelope so a shape change# surfaces as an empty sync rather than a crash deep in the pipeline.if not isinstance(data, list):@@ -147,6 +173,112 @@ def _fetch(return data+def _format_since(value: Any) -> str:+ if isinstance(value, datetime):+ return (value if value.tzinfo else value.replace(tzinfo=UTC)).isoformat()+ return str(value)+++def _job_fields(job: dict[str, Any]) -> dict[str, Any]:+ return {"workflow_id": job.get("workflow_id"), "job_created_at": job.get("created_at")}+++def _get_job_details(+ session: requests.Session,+ origin: str,+ config: UnstructuredEndpointConfig,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ for job in _fetch(session, f"{origin}{JOBS_PATH}", headers, None, logger):+ job_id = job.get("id")+ if not job_id:+ continue+ url = f"{origin}{config.path.format(job_id=job_id)}"+ details = _fetch_json(session, url, headers, None, logger, allow_not_found=True)+ if isinstance(details, dict):+ yield [{**details, **_job_fields(job)}]+++def _get_job_failed_files(+ session: requests.Session,+ origin: str,+ config: UnstructuredEndpointConfig,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ for job in _fetch(session, f"{origin}{JOBS_PATH}", headers, None, logger):+ job_id = job.get("id")+ # The job listing already counts failures, so a job with none needs no request. A null count+ # (older data planes) is fetched anyway.+ if not job_id or job.get("failed_file_count") == 0:+ continue+ url = f"{origin}{config.path.format(job_id=job_id)}"+ offset = 0+ while True:+ # Without paging params the API returns only a legacy sample of the failures.+ params = {"limit": FAILED_FILES_PAGE_SIZE, "offset": offset}+ data = _fetch_json(session, url, headers, params, logger, allow_not_found=True)+ if not isinstance(data, dict):+ break++ rows = [+ {+ **failed_file,+ "job_id": job_id,+ **_job_fields(job),+ "node_id": (failed_file.get("node") or {}).get("id"),+ }+ for failed_file in data.get("failed_files") or []+ if isinstance(failed_file, dict)+ ]+ if rows:+ yield rows++ next_offset = data.get("next_offset")+ if not isinstance(next_offset, int) or next_offset <= offset:+ break+ offset = next_offset+++def _get_notifications(+ session: requests.Session,+ url: str,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[UnstructuredResumeConfig],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+) -> Iterator[list[dict[str, Any]]]:+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ cursor = resume.next_cursor if resume else None++ base_params: dict[str, Any] = {"limit": NOTIFICATIONS_PAGE_SIZE}+ if should_use_incremental_field and db_incremental_field_last_value is not None:+ base_params["since"] = _format_since(db_incremental_field_last_value)++ while True:+ params = {**base_params, "cursor": cursor} if cursor else base_params+ data = _fetch_json(session, url, headers, params, logger)+ if not isinstance(data, dict):+ logger.warning(f"Unstructured API returned a non-object body for {url}; got {type(data).__name__}")+ return++ events = [event for event in data.get("events") or [] if isinstance(event, dict)]+ next_cursor = data.get("next_cursor")+ if next_cursor:+ resumable_source_manager.save_state(UnstructuredResumeConfig(next_cursor=next_cursor))++ if events:+ yield events+ elif next_cursor:+ resumable_source_manager.safe_point()++ if not next_cursor or next_cursor == cursor:+ return+ cursor = next_cursor++def get_rows(base_url: str | None,api_key: str,@@ -154,6 +286,8 @@ def get_rows(logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[UnstructuredResumeConfig],team_id: int,+ should_use_incremental_field: bool = False,+ db_incremental_field_last_value: Any = None,) -> Iterator[list[dict[str, Any]]]:# The API host is customer-controlled, so block hosts resolving to private/internal addresses# (SSRF) before sending the key. Only enforced on cloud — see `_is_host_safe`.@@ -163,7 +297,8 @@ def get_rows(config = UNSTRUCTURED_ENDPOINTS[endpoint]headers = _headers(api_key)- url = f"{normalize_base_url(base_url)}{config.path}"+ origin = normalize_base_url(base_url)+ url = f"{origin}{config.path}"# One session reused across pages so urllib3 keeps the connection alive between requests.# `allow_redirects=False` keeps the credentialed request pinned to the validated host so a# customer-controlled host can't 30x the `unstructured-api-key` header off-origin (SSRF). The@@ -175,6 +310,23 @@ def get_rows(redact_values=(api_key,), allow_redirects=False, capture=not bool(config.drop_fields))+ if config.fan_out_jobs:+ fan_out = _get_job_details if endpoint == "job_details" else _get_job_failed_files+ yield from fan_out(session, origin, config, headers, logger)+ return++ if endpoint == "notifications":+ yield from _get_notifications(+ session,+ url,+ headers,+ logger,+ resumable_source_manager,+ should_use_incremental_field,+ db_incremental_field_last_value,+ )+ return+if not config.paginated:rows = _drop_sensitive_fields(_fetch(session, url, headers, None, logger), config.drop_fields)if rows:@@ -219,6 +371,8 @@ def unstructured_source(logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[UnstructuredResumeConfig],team_id: int,+ should_use_incremental_field: bool = False,+ db_incremental_field_last_value: Any = None,) -> SourceResponse:config: UnstructuredEndpointConfig = UNSTRUCTURED_ENDPOINTS[endpoint]@@ -231,14 +385,18 @@ def unstructured_source(logger=logger,resumable_source_manager=resumable_source_manager,team_id=team_id,+ should_use_incremental_field=should_use_incremental_field,+ db_incremental_field_last_value=db_incremental_field_last_value,),primary_keys=config.primary_keys,- sort_mode="asc",+ sort_mode=config.sort_mode,partition_count=1,partition_size=1,partition_mode="datetime" if config.partition_key else None,partition_format="month" if config.partition_key else None,partition_keys=[config.partition_key] if config.partition_key else None,+ # The per-job walks keep no job or offset checkpoint, so a handoff would repeat every request.+ supports_resume=not config.fan_out_jobs,)