Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the Kisi source (#125361)

Tom Owers5617d770ea27
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 22db70f4ab77..eee527c2907a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -6872,16 +6872,16 @@ Note: Parsed the embedded OpenAPI document used by the official Swagger UI and e
## Kisi — gaps
-Today (9): `controllers`, `events`, `groups`, `locks`, `places`, `readers`, `role_assignments`, `roles`, `users`
+Today (12): `controllers`, `events`, `group_locks`, `groups`, `locks`, `places`, `readers`, `role_assignments`, `roles`, `team_memberships`, `teams`, `users`
Diffed against: <https://api.getkisi.com/openapi/apps.yaml>
- [x] `Roles (GET /roles)` — resolve role_id in role assignments and audit granted permissions (high) → `roles`
- [x] `Event sets (POST /event_sets; GET /event_sets/{id})` — analyze door access attempts, failures, and security activity missing from device and user inventories (high) → `events` (incremental on `created_at`, read in 90-day event set windows)
- [ ] `Presences (GET /presences)` — measure attendance and occupancy from first and last daily unlock activity (high); skipped: needs a `place_id` and a single `date` per request, and the first and last daily unlocks it reports can be derived from the `events` table
-- [ ] `Group locks (GET /group_locks)` — connect access groups to doors and audit effective access rights (medium)
-- [ ] `Team memberships (GET /team_memberships)` — expand access assigned to teams into individual users and guests (medium)
-- [ ] `Teams (GET /teams)` — analyze teams used as role assignees and join them to membership records (medium)
+- [x] `Group locks (GET /group_locks)` — connect access groups to doors and audit effective access rights (medium) → `group_locks` (fan-out over `groups`, because Kisi requires `group_id`)
+- [x] `Team memberships (GET /team_memberships)` — expand access assigned to teams into individual users and guests (medium) → `team_memberships`
+- [x] `Teams (GET /teams)` — analyze teams used as role assignees and join them to membership records (medium) → `teams`
- [ ] `Members (GET /members)` — analyze organization membership, access status, and links between members and user accounts (medium); skipped: Kisi marks the endpoint deprecated, and `users` already carries the same access fields
Note: Parsed the official OpenAPI document linked by the vendor API reference, including nested response schemas. Event exports require POST /event_sets, polling, and cursor pagination; each requested interval is limited to 90 days. Presences require a place and date, while role assignments already cover direct access assignments. Deprecated shares, login endpoints, device wiring, and settings were excluded.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/canonical_descriptions.py
index 09cdf621a202..9bd40fe41a4f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/canonical_descriptions.py
@@ -122,4 +122,38 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "Time when the event was created.",
},
},
+ "group_locks": {
+ "description": "Links between Kisi groups and the locks that their members can open.",
+ "docs_url": "https://api.getkisi.com/docs#tag/group-locks/GET/group_locks",
+ "columns": {
+ "id": "Unique group lock identifier.",
+ "group_id": "Identifier of the group that gets access to the lock.",
+ "lock_id": "Identifier of the lock that the group can open.",
+ "place_id": "Identifier of the place that contains the lock.",
+ "created_at": "Time when the group lock was created.",
+ "updated_at": "Time when the group lock was last updated.",
+ },
+ },
+ "teams": {
+ "description": "Teams of Kisi users and guests that receive roles together.",
+ "docs_url": "https://api.getkisi.com/docs#tag/teams/GET/teams",
+ "columns": {
+ "id": "Unique team identifier.",
+ "name": "Team name.",
+ "created_at": "Time when the team was created.",
+ "updated_at": "Time when the team was last updated.",
+ },
+ },
+ "team_memberships": {
+ "description": "Users and guests that belong to each Kisi team.",
+ "docs_url": "https://api.getkisi.com/docs#tag/team-memberships/GET/team_memberships",
+ "columns": {
+ "id": "Unique team membership identifier.",
+ "team_id": "Identifier of the team.",
+ "member_type": "Type of the member: User or Guest.",
+ "member_id": "Identifier of the user or guest.",
+ "created_at": "Time when the team membership was created.",
+ "updated_at": "Time when the team membership was last updated.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/kisi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/kisi.py
index 1ab65a95a186..a4db4ea3ce46 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/kisi.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/kisi.py
@@ -14,13 +14,14 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.int
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
- rest_api_resource,
+ rest_api_resources,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
OffsetPaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.kisi.settings import (
@@ -39,6 +40,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.kisi.setti
EVENTS_POLL_INTERVAL_SECONDS,
EVENTS_POLL_MAX_ATTEMPTS,
EVENTS_WINDOW,
+ GROUP_LOCKS,
+ GROUP_LOCKS_PATH,
+ GROUPS,
MAX_OFFSET,
OFFSET_ERROR,
PAGE_SIZE,
@@ -59,6 +63,7 @@ class KisiResumeConfig:
event_set_id: int | None = None
events_cursor: str | None = None
events_written: int = 0
+ fanout_state: dict[str, Any] | None = None
class KisiEventSetError(Exception):
@@ -125,10 +130,50 @@ def kisi_source(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ is_fanout = inputs.schema_name == GROUP_LOCKS
+
def save_checkpoint(state: dict[str, Any] | None) -> None:
- if state is not None:
+ if state is None:
+ return
+ if is_fanout:
+ resumable_source_manager.save_state(KisiResumeConfig(fanout_state=state))
+ else:
resumable_source_manager.save_state(KisiResumeConfig(offset=int(state["offset"])))
+ initial_paginator_state: dict[str, Any] | None = None
+ if resume is not None:
+ initial_paginator_state = resume.fanout_state if is_fanout else {"offset": resume.offset}
+
+ resources: list[str | EndpointResource] = (
+ [
+ {
+ "name": GROUPS,
+ "endpoint": {"path": GROUPS, "data_selector": "$"},
+ "write_disposition": "replace",
+ },
+ {
+ "name": GROUP_LOCKS,
+ "endpoint": {
+ "path": GROUP_LOCKS_PATH,
+ "params": {"group_id": {"type": "resolve", "resource": GROUPS, "field": "id"}},
+ "data_selector": "$",
+ # A path with a query string otherwise gets the single page paginator.
+ "paginator": KisiPaginator(),
+ # A group deleted after the groups page was read answers 404.
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
+ },
+ "write_disposition": "replace",
+ },
+ ]
+ if is_fanout
+ else [
+ {
+ "name": inputs.schema_name,
+ "endpoint": {"path": inputs.schema_name, "data_selector": "$"},
+ "write_disposition": "replace",
+ }
+ ]
+ )
config: RESTAPIConfig = {
"client": {
"base_url": BASE_URL,
@@ -138,21 +183,19 @@ def kisi_source(
"request_timeout": REQUEST_TIMEOUT,
"allow_redirects": False,
},
- "resources": [
- {
- "name": inputs.schema_name,
- "endpoint": {"path": inputs.schema_name, "data_selector": "$"},
- "write_disposition": "replace",
- }
- ],
+ "resources": resources,
}
- resource = rest_api_resource(
- config,
- inputs.team_id,
- inputs.job_id,
- None,
- resume_hook=save_checkpoint,
- initial_paginator_state={"offset": resume.offset} if resume else None,
+ resource = next(
+ item
+ for item in rest_api_resources(
+ config,
+ inputs.team_id,
+ inputs.job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+ if item.name == inputs.schema_name
)
return SourceResponse(
name=inputs.schema_name,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/settings.py
index f33066ca29c6..694befa01ff1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/settings.py
@@ -8,8 +8,23 @@ MAX_OFFSET = 20_000
EVENTS = "events"
ROLES = "roles"
+GROUPS = "groups"
+GROUP_LOCKS = "group_locks"
-ENDPOINTS = ("locks", "places", "users", "groups", "role_assignments", "controllers", "readers", ROLES, EVENTS)
+ENDPOINTS = (
+ "locks",
+ "places",
+ "users",
+ GROUPS,
+ "role_assignments",
+ "controllers",
+ "readers",
+ ROLES,
+ EVENTS,
+ GROUP_LOCKS,
+ "teams",
+ "team_memberships",
+)
PRIMARY_KEYS: dict[str, list[str]] = {EVENTS: ["uuid"]}
DEFAULT_PRIMARY_KEYS = ["id"]
# Roles have no creation timestamp to partition on.
@@ -17,6 +32,8 @@ PARTITION_KEYS: dict[str, list[str] | None] = {ROLES: None}
DEFAULT_PARTITION_KEYS = ["created_at"]
# Kisi returns every role in one response, with no pagination parameters.
UNPAGINATED_ENDPOINTS = (ROLES,)
+# Kisi requires a group_id on every group locks request, so the table reads each group in turn.
+GROUP_LOCKS_PATH = "group_locks?group_id={group_id}"
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
EVENTS: [
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/tests/test_kisi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/tests/test_kisi.py
index 04234fa59a4b..a0e257a6208a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/tests/test_kisi.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kisi/tests/test_kisi.py
@@ -10,6 +10,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
always,
scripted_network,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing.fake_network import RecordedRequest
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.kisi import KisiSourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.kisi.kisi import (
KisiEventSetError,
@@ -69,6 +70,50 @@ def test_pagination_and_resume(
assert result.saved_states == [KisiResumeConfig(offset=value) for value in expected_offsets[1:]]
+def _group_locks_responder(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/groups":
+ return ScriptedResponse(json=[{"id": 1}, {"id": 2}, {"id": 3}])
+ group_id, offset = request.param("group_id"), request.param("offset")
+ if group_id == "1":
+ size = 100 if offset == "0" else 1
+ return ScriptedResponse(json=[{"id": 1000 + int(offset or 0) + i, "group_id": 1} for i in range(size)])
+ if group_id == "2":
+ return ScriptedResponse(status=404, json={"error": "Not found", "code": "not_found"})
+ return ScriptedResponse(json=[{"id": 3000, "group_id": 3}])
+
+
[email protected]("resumed", [False, True])
+def test_group_locks_read_each_group(resumed: bool, driver: SourceDriver) -> None:
+ result = driver.run(
+ "group_locks",
+ _group_locks_responder,
+ resume_state=KisiResumeConfig(
+ fanout_state={"completed": [], "current": "group_locks?group_id=1", "child_state": {"offset": 100}}
+ )
+ if resumed
+ else None,
+ )
+
+ first_group_offsets = ["100"] if resumed else ["0", "100"]
+ assert result.raised is None
+ assert [row["id"] for row in result.rows] == [
+ 1000 + int(offset) + i for offset in first_group_offsets for i in range(100 if offset == "0" else 1)
+ ] + [3000]
+ assert [(request.path, request.param("group_id"), request.param("offset")) for request in result.requests] == [
+ ("/groups", None, "0"),
+ *[("/group_locks", "1", offset) for offset in first_group_offsets],
+ ("/group_locks", "2", "0"),
+ ("/group_locks", "3", "0"),
+ ]
+ if not resumed:
+ assert (
+ KisiResumeConfig(
+ fanout_state={"completed": [], "current": "group_locks?group_id=1", "child_state": {"offset": 100}}
+ )
+ in result.saved_states
+ )
+
+
@pytest.mark.parametrize("total,should_fail", [(20_100, False), (20_101, True)])
def test_offset_limit_does_not_silently_truncate(total: int, should_fail: bool, driver: SourceDriver) -> None:
rows: list[dict[str, object]] = [{"id": 20_000 + i} for i in range(100)]