Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the Unstructured source (#119732)

Tom Owers3877eabcdd33
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index a4c7aea359d6..639913b96f4f 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1166,8 +1166,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/uk_companies_house/uk_companies_house.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/unleash/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/unleash/unleash.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/uppromote.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/upstash/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/uptimerobot/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 70630ca942c4..82008cf78a39 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -12762,9 +12762,9 @@ Today (4): `destinations`, `jobs`, `sources`, `workflows`
Diffed against: <https://docs.unstructured.io/llms.txt>
-- [ ] `job failed files (GET /jobs/{id}/failed-files)` — per-file failure reasons — the only place processing errors are enumerated; jobs alone give just a status (high)
-- [ ] `job details (GET /jobs/{id}/details)` — per-node processing counts and timings inside a job, the grain needed for throughput and cost analysis (high)
-- [ ] `notifications (GET /notifications)` — workspace event stream (job started/failed/completed) with timestamps, an analytical log the jobs table does not reproduce (medium)
+- [x] `job failed files (GET /jobs/{id}/failed-files)` — per-file failure reasons — the only place processing errors are enumerated; jobs alone give just a status (high)
+- [x] `job details (GET /jobs/{id}/details)` — per-node processing counts and timings inside a job, the grain needed for throughput and cost analysis (high)
+- [x] `notifications (GET /notifications)` — workspace event stream (job started/failed/completed) with timestamps, an analytical log the jobs table does not reproduce (medium)
- [ ] `templates (GET /templates)` — lookup resolving the template a workflow was created from, plus its DAG metadata (medium)
- [ ] `source/destination connection checks (GET /sources/{id}/connection-check, /destinations/{id}/connection-check)` — connector health history to correlate with failed jobs (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/canonical_descriptions.py
index 3b13a7997ebc..d472fe43da2e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/canonical_descriptions.py
@@ -64,4 +64,45 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"key": "Optional stable key for the destination connector.",
},
},
+ "job_details": {
+ "description": "Processing details of each job: per-node document counts, overall outcome, and progress.",
+ "docs_url": "https://docs.unstructured.io/api-reference/workflow/jobs",
+ "columns": {
+ "id": "Unique identifier of the job.",
+ "workflow_id": "Identifier of the workflow this job ran.",
+ "job_created_at": "Timestamp when the job was created.",
+ "processing_status": "Document-level outcome of the job (e.g. SUCCESS, COMPLETED_WITH_ERRORS, FAILED).",
+ "node_stats": "Per-node document counts (ready, in progress, success, failure) for each step of the workflow DAG.",
+ "summary": "Totals of documents processed, succeeded, and failed across the job.",
+ "message": "Human-readable message about the job's processing state.",
+ "progress": "Advisory reading of why the job is not finished yet, including when its steps last reported.",
+ "queued_behind_job_ids_snapshot": "Jobs that held the workspace's concurrency slots when this job was queued.",
+ "completion_issue": "Why a job completed without delivering a document, if an issue was detected.",
+ },
+ },
+ "job_failed_files": {
+ "description": "Files that failed to process in a job, with the workflow node that failed each one and the error.",
+ "docs_url": "https://docs.unstructured.io/api-reference/workflow/jobs",
+ "columns": {
+ "job_id": "Identifier of the job the file failed in.",
+ "workflow_id": "Identifier of the workflow the job ran.",
+ "job_created_at": "Timestamp when the job was created.",
+ "document": "The file that failed to process.",
+ "error": "Error message for the failure.",
+ "node": "The workflow node that failed the file (id, name, type, and subtype).",
+ "node_id": "Identifier of the workflow node that failed the file.",
+ },
+ },
+ "notifications": {
+ "description": "Platform notification events, such as a job that started, completed, or failed.",
+ "docs_url": "https://docs.unstructured.io/api-reference",
+ "columns": {
+ "id": "Unique identifier of the notification event.",
+ "event_type": "Event type (e.g. job.completed, job.failed).",
+ "workflow_id": "Workflow the event relates to; null for account-level events.",
+ "payload": "Event payload data.",
+ "created_at": "Timestamp when the event was created.",
+ "is_read": "Whether the user that owns the API key marked the notification as read.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/posthog_com_doc.mdx b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/posthog_com_doc.mdx
index 674c80c580fd..cd6da2311a30 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/posthog_com_doc.mdx
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/posthog_com_doc.mdx
@@ -22,7 +22,7 @@ import AlphaRelease from '../_snippets/alpha-release.mdx'
<AlphaRelease />
-[Unstructured](https://unstructured.io) is a document ETL platform that turns raw files (PDFs, Office documents, emails, and more) into RAG-ready structured data. This source pulls the metadata from your Unstructured platform account, its workflows, job runs, and source/destination connectors, into the PostHog data warehouse so you can monitor the health of your document pipelines alongside the rest of your data.
+[Unstructured](https://unstructured.io) is a document ETL platform that turns raw files (PDFs, Office documents, emails, and more) into RAG-ready structured data. This source pulls the metadata from your Unstructured platform account, its workflows, job runs, per-job processing details and failed files, notifications, and source/destination connectors, into the PostHog data warehouse so you can monitor the health of your document pipelines alongside the rest of your data.
## Prerequisites
@@ -45,7 +45,9 @@ Leave **API host** blank unless Unstructured provisioned your account with a cus
<SyncModes />
-Unstructured's read endpoints do not expose a reliable server-side incremental filter, and workflows and connectors are mutable configuration objects, so every table syncs as a full refresh. These are low-volume inventory tables, so a full refresh each sync is inexpensive and always reflects the current state.
+The `notifications` table supports incremental sync on `created_at`. Every other table syncs as a full refresh: workflows and connectors are mutable configuration objects, and jobs have no server-side time filter. These are low-volume inventory tables, so a full refresh each sync always reflects the current state.
+
+The `job_details` and `job_failed_files` tables make one request for each job on every sync. `job_failed_files` skips jobs that report no failed files. `job_details` is off by default, so turn it on only when you need per-node processing counts.
## Configuration
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py
index fe39dc6ef566..e2f8149f872b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/settings.py
@@ -1,21 +1,24 @@
from dataclasses import dataclass, field
+from typing import Literal
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-# Unstructured's Platform API paths always carry a trailing slash; hitting them without it returns a
-# 307 redirect (which also downgrades the scheme), so keep the slash to talk to the API directly.
+# Paths match the Platform API spec exactly: collection paths carry a trailing slash, and the per-job
+# and notification paths do not. A collection path without its slash returns a 307 redirect (which
+# also downgrades the scheme), so keep each path as the spec writes it.
-@dataclass
+@dataclass(frozen=True)
class UnstructuredEndpointConfig:
name: str
path: str
# Field to partition Delta files by. Must be stable (never rewritten upstream) so partitions
# don't churn every sync; `created_at` is set once at object creation across every resource here.
+ # The per-job tables carry no timestamp of their own, so they partition on the parent job's.
partition_key: str | None = "created_at"
primary_keys: list[str] = field(default_factory=lambda: ["id"])
- # Only /workflows/ accepts page / page_size; the other list endpoints return the full array in a
- # single response, so pagination (and resume) only applies to workflows.
+ # Only /workflows/ accepts page / page_size. /notifications and the per-job failed files have their
+ # own cursor and offset paging; the other list endpoints return the full array in one response.
paginated: bool = False
should_sync_default: bool = True
# Top-level fields dropped from every row before it is yielded. The /sources/ and /destinations/
@@ -24,6 +27,9 @@ class UnstructuredEndpointConfig:
# access could read it back. We drop the whole object rather than allowlisting because the fields
# vary per connector type and a new type could introduce a secret we don't yet know to strip.
drop_fields: tuple[str, ...] = ()
+ # Per-job endpoints: list /jobs/ first, then call `path` (with a `{job_id}` placeholder) once per job.
+ fan_out_jobs: bool = False
+ sort_mode: Literal["asc", "desc"] = "asc"
UNSTRUCTURED_ENDPOINTS: dict[str, UnstructuredEndpointConfig] = {
@@ -33,14 +39,41 @@ UNSTRUCTURED_ENDPOINTS: dict[str, UnstructuredEndpointConfig] = {
"destinations": UnstructuredEndpointConfig(
name="destinations", path="/api/v1/destinations/", drop_fields=("config",)
),
+ # One request per job on every sync, so it is off by default to keep the request volume opt-in.
+ "job_details": UnstructuredEndpointConfig(
+ name="job_details",
+ path="/api/v1/jobs/{job_id}/details",
+ partition_key="job_created_at",
+ fan_out_jobs=True,
+ should_sync_default=False,
+ ),
+ # A job reports each failure once per file and workflow node, so neither field is unique alone.
+ "job_failed_files": UnstructuredEndpointConfig(
+ name="job_failed_files",
+ path="/api/v1/jobs/{job_id}/failed-files",
+ partition_key="job_created_at",
+ primary_keys=["job_id", "node_id", "document"],
+ fan_out_jobs=True,
+ ),
+ # The API returns notifications newest first and accepts no sort parameter.
+ "notifications": UnstructuredEndpointConfig(name="notifications", path="/api/v1/notifications", sort_mode="desc"),
}
ENDPOINTS = tuple(UNSTRUCTURED_ENDPOINTS.keys())
-# All four endpoints ship as full refresh. Only /workflows/ exposes a server-side timestamp filter
-# (`created_since`); jobs, sources, and destinations have no server-side time filter at all (confirmed
-# against the Platform API OpenAPI spec), so a "since" sync would still page the whole list and isn't
-# genuinely incremental. Workflows, sources, and destinations are also low-volume mutable config
-# inventory where a full refresh yields the correct current state (status/schedule edits included),
-# which incremental-on-created_at would freeze. See the module docstring in `unstructured.py`.
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+# Only notifications sync incrementally: they are an append-only event log with a server-side `since`
+# filter on `created_at`. /workflows/ also has a `created_since` filter, but workflows, sources, and
+# destinations are low-volume mutable config inventory where a full refresh yields the correct current
+# state (status/schedule edits included), which incremental-on-created_at would freeze. Jobs and the
+# per-job tables have no server-side time filter at all (confirmed against the Platform API OpenAPI
+# spec), so a "since" sync would still page the whole list. See the module docstring in `unstructured.py`.
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ "notifications": [
+ {
+ "label": "created_at",
+ "type": IncrementalFieldType.DateTime,
+ "field": "created_at",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/source.py
index 0bd42cd47c07..98985a568a5d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/source.py
@@ -157,4 +157,6 @@ Leave **API host** blank unless Unstructured provisioned your account with a cus
logger=inputs.logger,
resumable_source_manager=resumable_source_manager,
team_id=inputs.team_id,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/tests/test_unstructured.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/tests/test_unstructured.py
index 4ddd4d69a8cd..e3615a222505 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/tests/test_unstructured.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/tests/test_unstructured.py
@@ -1,3 +1,6 @@
+from datetime import UTC, datetime
+from typing import Any
+
from unittest.mock import patch
import requests
@@ -7,6 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.unstructured import (
@@ -15,6 +19,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
from products.warehouse_sources.backend.temporal.data_imports.sources.unstructured.source import UnstructuredSource
from products.warehouse_sources.backend.temporal.data_imports.sources.unstructured.unstructured import (
DEFAULT_BASE_URL,
+ FAILED_FILES_PAGE_SIZE,
+ NOTIFICATIONS_PAGE_SIZE,
WORKFLOWS_PAGE_SIZE,
UnstructuredResumeConfig,
UnstructuredRetryableError,
@@ -161,6 +167,183 @@ class TestGetRows:
assert result.requests == []
+JOB_A = "11111111-1111-4111-8111-111111111111"
+JOB_B = "22222222-2222-4222-8222-222222222222"
+JOB_C = "33333333-3333-4333-8333-333333333333"
+
+
+def _job(job_id: str, failed_file_count: int | None = None) -> dict[str, Any]:
+ return {
+ "id": job_id,
+ "workflow_id": "wf-1",
+ "created_at": "2026-01-02T03:04:05Z",
+ "status": "COMPLETED",
+ "failed_file_count": failed_file_count,
+ }
+
+
+def _failed_file(document: str, node_id: str = "node-partition") -> dict[str, Any]:
+ return {"document": document, "error": "could not parse", "node": {"id": node_id, "type": "partition"}}
+
+
+class TestJobFanOut:
+ def test_failed_files_skips_clean_jobs_and_walks_offset_pages(self) -> None:
+ result = _driver().run(
+ "job_failed_files",
+ route(
+ {
+ "/api/v1/jobs/": [ScriptedResponse(json=[_job(JOB_A, 0), _job(JOB_B, 3), _job(JOB_C)])],
+ f"/api/v1/jobs/{JOB_B}/failed-files": [
+ ScriptedResponse(
+ json={"failed_files": [_failed_file("a.pdf"), _failed_file("b.pdf")], "next_offset": 2}
+ ),
+ ScriptedResponse(json={"failed_files": [_failed_file("c.pdf")], "next_offset": None}),
+ ],
+ f"/api/v1/jobs/{JOB_C}/failed-files": [ScriptedResponse(status=404, json={"detail": "gone"})],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert f"/api/v1/jobs/{JOB_A}/failed-files" not in result.paths
+ assert result.params("offset") == [None, "0", "2", "0"]
+ assert result.params("limit")[1:] == [str(FAILED_FILES_PAGE_SIZE)] * 3
+ assert [row["document"] for row in result.rows] == ["a.pdf", "b.pdf", "c.pdf"]
+ assert result.rows[0] == {
+ **_failed_file("a.pdf"),
+ "job_id": JOB_B,
+ "workflow_id": "wf-1",
+ "job_created_at": "2026-01-02T03:04:05Z",
+ "node_id": "node-partition",
+ }
+
+ @parameterized.expand(
+ [
+ ("no_next_offset", {"failed_files": [{"document": "a.pdf", "error": "x", "node": {"id": "n"}}]}),
+ ("next_offset_not_ahead", {"failed_files": [], "next_offset": 0}),
+ ]
+ )
+ def test_failed_files_stops_without_a_forward_offset(self, _name: str, body: dict[str, Any]) -> None:
+ result = _driver().run(
+ "job_failed_files",
+ route(
+ {
+ "/api/v1/jobs/": [ScriptedResponse(json=[_job(JOB_A, 1)])],
+ f"/api/v1/jobs/{JOB_A}/failed-files": [ScriptedResponse(json=body)],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert len(result.requests) == 2
+
+ def test_job_details_adds_job_fields_and_skips_deleted_jobs(self) -> None:
+ details = {"id": JOB_A, "processing_status": "SUCCESS", "node_stats": [{"node_id": "n", "success": 4}]}
+ result = _driver().run(
+ "job_details",
+ route(
+ {
+ "/api/v1/jobs/": [ScriptedResponse(json=[_job(JOB_A), _job(JOB_B)])],
+ f"/api/v1/jobs/{JOB_A}/details": [ScriptedResponse(json=details)],
+ f"/api/v1/jobs/{JOB_B}/details": [ScriptedResponse(status=404, json={"detail": "gone"})],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.rows == [{**details, "workflow_id": "wf-1", "job_created_at": "2026-01-02T03:04:05Z"}]
+
+ def test_per_job_server_error_still_fails_the_sync(self) -> None:
+ # Only a 404 means the job is gone; any other error must not silently drop the job's rows.
+ result = _driver().run(
+ "job_details",
+ route(
+ {
+ "/api/v1/jobs/": [ScriptedResponse(json=[_job(JOB_A)])],
+ f"/api/v1/jobs/{JOB_A}/details": [ScriptedResponse(status=403, json={"detail": "no"})],
+ }
+ ),
+ )
+
+ assert isinstance(result.raised, requests.HTTPError)
+
+
+def _event(event_id: str, created_at: str = "2026-03-01T00:00:00Z") -> dict[str, Any]:
+ return {"id": event_id, "event_type": "job.completed", "payload": {}, "created_at": created_at}
+
+
+class TestNotifications:
+ def test_walks_cursor_pages_and_stages_each_cursor_before_its_page(self) -> None:
+ result = _driver().run(
+ "notifications",
+ [
+ ScriptedResponse(json={"events": [_event("e1"), _event("e2")], "next_cursor": "c1"}),
+ ScriptedResponse(json={"events": [], "next_cursor": "c2"}),
+ ScriptedResponse(json={"events": [_event("e3")], "next_cursor": None}),
+ ],
+ )
+
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == ["e1", "e2", "e3"]
+ assert result.params("cursor") == [None, "c1", "c2"]
+ assert result.params("limit") == [str(NOTIFICATIONS_PAGE_SIZE)] * 3
+ assert result.saved_states == [
+ UnstructuredResumeConfig(next_cursor="c1"),
+ UnstructuredResumeConfig(next_cursor="c2"),
+ ]
+ assert result.committed_states == result.saved_states
+ assert result.response is not None and result.response.sort_mode == "desc"
+
+ def test_resumes_from_saved_cursor(self) -> None:
+ result = _driver().run(
+ "notifications",
+ [ScriptedResponse(json={"events": [_event("e9")], "next_cursor": None})],
+ resume_state=UnstructuredResumeConfig(next_cursor="c7"),
+ )
+
+ assert result.params("cursor") == ["c7"]
+ assert [row["id"] for row in result.rows] == ["e9"]
+
+ def test_repeated_cursor_ends_the_walk(self) -> None:
+ result = _driver().run(
+ "notifications",
+ [
+ ScriptedResponse(json={"events": [_event("e1")], "next_cursor": "c1"}),
+ ScriptedResponse(json={"events": [_event("e1")], "next_cursor": "c1"}),
+ ],
+ )
+
+ assert result.raised is None
+ assert len(result.requests) == 2
+
+ @parameterized.expand(
+ [
+ ("incremental_datetime", True, datetime(2026, 2, 1, 12, 0), "2026-02-01T12:00:00+00:00"),
+ ("incremental_string", True, "2026-02-01T12:00:00Z", "2026-02-01T12:00:00Z"),
+ ("incremental_first_sync", True, None, None),
+ ("full_refresh_ignores_watermark", False, datetime(2026, 2, 1, tzinfo=UTC), None),
+ ]
+ )
+ def test_since_follows_the_incremental_watermark(
+ self, _name: str, incremental: bool, last_value: Any, expected_since: str | None
+ ) -> None:
+ result = _driver().run(
+ "notifications",
+ [ScriptedResponse(json={"events": [], "next_cursor": None})],
+ incremental_field="created_at",
+ should_use_incremental_field=incremental,
+ db_incremental_field_last_value=last_value,
+ )
+
+ assert result.params("since") == [expected_since]
+
+ def test_non_object_body_returns_empty(self) -> None:
+ result = _driver().run("notifications", [ScriptedResponse(json=[])])
+
+ assert result.raised is None
+ assert result.rows == []
+
+
class TestUnstructuredSource:
@parameterized.expand(["workflows", "jobs", "sources", "destinations"])
def test_source_response_contract(self, endpoint: str) -> None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py
index 153edd4ca9a5..a29f0db96955 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py
@@ -5,22 +5,28 @@ list endpoints are synced: `/api/v1/workflows/`, `/api/v1/jobs/`, `/api/v1/sourc
`/api/v1/destinations/`. Each returns a bare JSON array of objects (no envelope). Only `/workflows/`
is paginated (page / page_size); the rest return their whole list in one response.
-Incremental sync is intentionally not offered. Only `/workflows/` exposes a server-side timestamp
-filter (`created_since`), and even there the resource is mutable config (status / schedule / updated_at
-change over time), so filtering on `created_at` would freeze already-synced rows and miss updates. The
-jobs / sources / destinations endpoints expose no server-side time filter at all, so a "since" sync
-would re-page the full list anyway. These are all low-volume inventory tables, so every stream ships as
-a full refresh, which reflects the correct current state each sync. The `created_since` filter and
-per-endpoint behavior were verified against the live Platform API OpenAPI spec (version 3.1.1); they
-were not smoke-tested against a real key, so a future implementer enabling incremental should confirm
-`created_since` actually filters before relying on it. The workflows `sort_by=created_at` below is
-similarly taken from the spec (sort_by is a free-form string defaulting to `id`) but unverified against
-a live key; if the API rejects it, drop the sort params and fall back to the default `id` ordering.
+Two per-job tables fan out over `/api/v1/jobs/`: `/jobs/{id}/details` (one object per job) and
+`/jobs/{id}/failed-files` (offset paging with `next_offset`). `/api/v1/notifications` is an event log
+wrapped in `{"events": [...], "next_cursor": ...}`, ordered newest first with cursor paging.
+
+Incremental sync is offered only for notifications, through the server-side `since` filter. Only
+`/workflows/` also exposes a server-side timestamp filter (`created_since`), and there the resource is
+mutable config (status / schedule / updated_at change over time), so filtering on `created_at` would
+freeze already-synced rows and miss updates. The jobs / sources / destinations endpoints expose no
+server-side time filter at all, so a "since" sync would re-page the full list anyway. These are all
+low-volume inventory tables, so they ship as a full refresh, which reflects the correct current state
+each sync. The filters and per-endpoint behavior were verified against the live Platform API OpenAPI
+spec (versions 3.1.1 and 3.3.0); they were not smoke-tested against a real key, so a future implementer
+enabling more incremental tables should confirm `created_since` actually filters before relying on it.
+The workflows `sort_by=created_at` below is similarly taken from the spec (sort_by is a free-form string
+defaulting to `id`) but unverified against a live key; if the API rejects it, drop the sort params and
+fall back to the default `id` ordering.
"""
import re
import dataclasses
from collections.abc import Iterator
+from datetime import UTC, datetime
from typing import Any, Optional
from urllib.parse import urlparse
@@ -40,6 +46,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.unstructur
DEFAULT_BASE_URL = "https://platform.unstructuredapp.io"
# /workflows/ default page_size is 20; pull larger pages to cut round-trips on busy accounts.
WORKFLOWS_PAGE_SIZE = 100
+# The largest page sizes the spec allows for these endpoints.
+NOTIFICATIONS_PAGE_SIZE = 100
+FAILED_FILES_PAGE_SIZE = 1000
+JOBS_PATH = "/api/v1/jobs/"
REQUEST_TIMEOUT = 60
HOST_NOT_ALLOWED_ERROR = "This API host is not allowed"
@@ -48,11 +58,12 @@ class UnstructuredRetryableError(Exception):
pass
[email protected](frozen=True)
class UnstructuredResumeConfig:
- # 1-based index of the next /workflows/ page to fetch. Only workflows paginate, so this is the
- # sole piece of resumable state.
- next_page: int
+ # 1-based index of the next /workflows/ page to fetch.
+ next_page: int = 1
+ # Cursor of the next /notifications page.
+ next_cursor: str | None = None
def normalize_base_url(base_url: str | None) -> str:
@@ -112,13 +123,14 @@ def _drop_sensitive_fields(rows: list[dict[str, Any]], drop_fields: tuple[str, .
wait=wait_exponential_jitter(initial=1, max=30),
reraise=True,
)
-def _fetch(
+def _fetch_json(
session: requests.Session,
url: str,
headers: dict[str, str],
params: Optional[dict[str, Any]],
logger: FilteringBoundLogger,
-) -> list[dict[str, Any]]:
+ allow_not_found: bool = False,
+) -> Any:
response = session.get(url, headers=headers, params=params, timeout=REQUEST_TIMEOUT)
# The session never follows redirects (SSRF hardening); surface a 30x as an error rather than
@@ -134,11 +146,25 @@ def _fetch(
f"Unstructured API error (retryable): status={response.status_code}, url={url}"
)
+ # A job deleted between the /jobs/ listing and its per-job request has nothing left to sync.
+ if allow_not_found and response.status_code == 404:
+ return None
+
if not response.ok:
logger.error(f"Unstructured API error: status={response.status_code}, body={response.text}, url={url}")
response.raise_for_status()
- data = response.json()
+ return response.json()
+
+
+def _fetch(
+ session: requests.Session,
+ url: str,
+ headers: dict[str, str],
+ params: Optional[dict[str, Any]],
+ logger: FilteringBoundLogger,
+) -> list[dict[str, Any]]:
+ data = _fetch_json(session, url, headers, params, logger)
# Every list endpoint returns a bare array; guard against an unexpected envelope so a shape change
# surfaces as an empty sync rather than a crash deep in the pipeline.
if not isinstance(data, list):
@@ -147,6 +173,112 @@ def _fetch(
return data
+def _format_since(value: Any) -> str:
+ if isinstance(value, datetime):
+ return (value if value.tzinfo else value.replace(tzinfo=UTC)).isoformat()
+ return str(value)
+
+
+def _job_fields(job: dict[str, Any]) -> dict[str, Any]:
+ return {"workflow_id": job.get("workflow_id"), "job_created_at": job.get("created_at")}
+
+
+def _get_job_details(
+ session: requests.Session,
+ origin: str,
+ config: UnstructuredEndpointConfig,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ for job in _fetch(session, f"{origin}{JOBS_PATH}", headers, None, logger):
+ job_id = job.get("id")
+ if not job_id:
+ continue
+ url = f"{origin}{config.path.format(job_id=job_id)}"
+ details = _fetch_json(session, url, headers, None, logger, allow_not_found=True)
+ if isinstance(details, dict):
+ yield [{**details, **_job_fields(job)}]
+
+
+def _get_job_failed_files(
+ session: requests.Session,
+ origin: str,
+ config: UnstructuredEndpointConfig,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ for job in _fetch(session, f"{origin}{JOBS_PATH}", headers, None, logger):
+ job_id = job.get("id")
+ # The job listing already counts failures, so a job with none needs no request. A null count
+ # (older data planes) is fetched anyway.
+ if not job_id or job.get("failed_file_count") == 0:
+ continue
+ url = f"{origin}{config.path.format(job_id=job_id)}"
+ offset = 0
+ while True:
+ # Without paging params the API returns only a legacy sample of the failures.
+ params = {"limit": FAILED_FILES_PAGE_SIZE, "offset": offset}
+ data = _fetch_json(session, url, headers, params, logger, allow_not_found=True)
+ if not isinstance(data, dict):
+ break
+
+ rows = [
+ {
+ **failed_file,
+ "job_id": job_id,
+ **_job_fields(job),
+ "node_id": (failed_file.get("node") or {}).get("id"),
+ }
+ for failed_file in data.get("failed_files") or []
+ if isinstance(failed_file, dict)
+ ]
+ if rows:
+ yield rows
+
+ next_offset = data.get("next_offset")
+ if not isinstance(next_offset, int) or next_offset <= offset:
+ break
+ offset = next_offset
+
+
+def _get_notifications(
+ session: requests.Session,
+ url: str,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[UnstructuredResumeConfig],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> Iterator[list[dict[str, Any]]]:
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ cursor = resume.next_cursor if resume else None
+
+ base_params: dict[str, Any] = {"limit": NOTIFICATIONS_PAGE_SIZE}
+ if should_use_incremental_field and db_incremental_field_last_value is not None:
+ base_params["since"] = _format_since(db_incremental_field_last_value)
+
+ while True:
+ params = {**base_params, "cursor": cursor} if cursor else base_params
+ data = _fetch_json(session, url, headers, params, logger)
+ if not isinstance(data, dict):
+ logger.warning(f"Unstructured API returned a non-object body for {url}; got {type(data).__name__}")
+ return
+
+ events = [event for event in data.get("events") or [] if isinstance(event, dict)]
+ next_cursor = data.get("next_cursor")
+ if next_cursor:
+ resumable_source_manager.save_state(UnstructuredResumeConfig(next_cursor=next_cursor))
+
+ if events:
+ yield events
+ elif next_cursor:
+ resumable_source_manager.safe_point()
+
+ if not next_cursor or next_cursor == cursor:
+ return
+ cursor = next_cursor
+
+
def get_rows(
base_url: str | None,
api_key: str,
@@ -154,6 +286,8 @@ def get_rows(
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[UnstructuredResumeConfig],
team_id: int,
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
) -> Iterator[list[dict[str, Any]]]:
# The API host is customer-controlled, so block hosts resolving to private/internal addresses
# (SSRF) before sending the key. Only enforced on cloud — see `_is_host_safe`.
@@ -163,7 +297,8 @@ def get_rows(
config = UNSTRUCTURED_ENDPOINTS[endpoint]
headers = _headers(api_key)
- url = f"{normalize_base_url(base_url)}{config.path}"
+ origin = normalize_base_url(base_url)
+ url = f"{origin}{config.path}"
# One session reused across pages so urllib3 keeps the connection alive between requests.
# `allow_redirects=False` keeps the credentialed request pinned to the validated host so a
# customer-controlled host can't 30x the `unstructured-api-key` header off-origin (SSRF). The
@@ -175,6 +310,23 @@ def get_rows(
redact_values=(api_key,), allow_redirects=False, capture=not bool(config.drop_fields)
)
+ if config.fan_out_jobs:
+ fan_out = _get_job_details if endpoint == "job_details" else _get_job_failed_files
+ yield from fan_out(session, origin, config, headers, logger)
+ return
+
+ if endpoint == "notifications":
+ yield from _get_notifications(
+ session,
+ url,
+ headers,
+ logger,
+ resumable_source_manager,
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ )
+ return
+
if not config.paginated:
rows = _drop_sensitive_fields(_fetch(session, url, headers, None, logger), config.drop_fields)
if rows:
@@ -219,6 +371,8 @@ def unstructured_source(
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[UnstructuredResumeConfig],
team_id: int,
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
config: UnstructuredEndpointConfig = UNSTRUCTURED_ENDPOINTS[endpoint]
@@ -231,14 +385,18 @@ def unstructured_source(
logger=logger,
resumable_source_manager=resumable_source_manager,
team_id=team_id,
+ should_use_incremental_field=should_use_incremental_field,
+ db_incremental_field_last_value=db_incremental_field_last_value,
),
primary_keys=config.primary_keys,
- sort_mode="asc",
+ sort_mode=config.sort_mode,
partition_count=1,
partition_size=1,
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
+ # The per-job walks keep no job or offset checkpoint, so a handoff would repeat every request.
+ supports_resume=not config.fan_out_jobs,
)