:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(integrations): add usage and latency metrics to the integrations SDK (#126259)
Tom Owersa49d21bd9b61
diff --git a/posthog/integrations/README.md b/posthog/integrations/README.mdindex 6a2c9c3c0215..b655c64cf0a3 100644--- a/posthog/integrations/README.md+++ b/posthog/integrations/README.md@@ -142,6 +142,28 @@ The default of a kind is stored data. Do not change the default of a kind that hWarehouse source kinds are not in the catalogue. The warehouse product registers them at startup with `register_source_kinds`, and callers reach their rows through `IntegrationManager.source(kind)`.A warehouse source row has `config["warehouse_source_type"]`. Every other accessor skips these rows unless you call `with_source_rows()`, and `IntegrationManager.source(kind)` reads only these rows.+## Metrics++`observability.py` records Prometheus metrics for every operation.+The `instrumented` decorator on each accessor method does this, so a new operation needs only the decorator.+An async form counts as the operation it calls.+No label holds a team id, an integration id, an external id or a config value.+A failure in the metric code is logged and never reaches the caller.++| Metric | Answers |+| ---------------------------------------------------------------------------------- | ----------------------------------------------------------------------- |+| `integrations_sdk_operations_total{operation, accessor, kind, auth_type, outcome}` | Which operations, accessors, kinds and auth types callers use |+| `integrations_sdk_operation_duration_seconds{operation, accessor, outcome}` | How long each operation takes |+| `integrations_sdk_rows_returned{operation, accessor}` | How many rows `list`, `get_many` and `list_by_external_id` return |+| `integrations_sdk_rows_written_total{operation, kind}` | How many rows `update_by_external_id` and `delete_by_external_id` touch |+| `integrations_sdk_token_refresh_total{kind, result}` | How often a refresh is needed, and how often it fails |++- `accessor` is `named` (one kind), `any` (`IntegrationManager.any`), `kinds` (`IntegrationManager.kinds(...)`) or `source` (`IntegrationManager.source(kind)`).+- `kind` is `multi` for `any` and `kinds`.+- `auth_type` is `oauth`, `token`, `credentials` or `any`.+- `outcome` is `ok`, `not_found`, `conflict`, `in_use`, `unsupported_filter`, `unsupported_auth_type` or `error`. `error` is any other exception, which the SDK raises again unchanged.+- `result` is `fresh` (no refresh needed), `refreshed`, `stale_token` (refreshed because the caller passed the stored token) or `error` (the refresh failed).+## Report usagesA product that uses integration rows reports them from `products/<product>/backend/integration_usages.py`:diff --git a/posthog/integrations/accessors.py b/posthog/integrations/accessors.pyindex e113cef98834..5a15bc64cac1 100644--- a/posthog/integrations/accessors.py+++ b/posthog/integrations/accessors.py@@ -37,6 +37,7 @@ from posthog.integrations.contracts import (UnsupportedIntegrationFilter,)from posthog.integrations.enums import AuthType+from posthog.integrations.observability import instrumented, record_token_refreshfrom posthog.integrations.source_kinds import is_source_kindfrom posthog.integrations.usages import integration_usagesfrom posthog.models.integration.auth_types import ANY, AUTH_TYPE_CONFIG_KEY, default_auth_type, kind_auth_types@@ -207,14 +208,17 @@ class IntegrationReader:except (Integration.DoesNotExist, ValueError, TypeError) as error:raise IntegrationNotFound(f"No integration {id!r} in {self!r} for team {team_id}") from error+ @instrumented("get")def get(self, team_id: int, id: int) -> IntegrationRecord:"""The row with this id. Raises `IntegrationNotFound` when the row is not in the scope."""return self._row(team_id, id)+ @instrumented("get_many", rows="returned")def get_many(self, team_id: int, ids: Iterable[int]) -> dict[int, IntegrationRecord]:"""The rows with these ids, keyed by id. Ids that are not in the scope are missing from the result."""return {row.id: row for row in self._team_rows(team_id).filter(id__in=list(ids))}+ @instrumented("find")def find(self,team_id: int | None = None,@@ -229,6 +233,7 @@ class IntegrationReader:"""return self._matching(team_id, external_id, where, team_ids).order_by("created_at", "id").first()+ @instrumented("list", rows="returned")def list(self,team_id: int,@@ -240,6 +245,7 @@ class IntegrationReader:order = ("-created_at", "-id") if newest_first else ("created_at", "id")return list(self._matching(team_id, external_id, where).order_by(*order))+ @instrumented("exists")def exists(self,team_id: int | None = None,@@ -251,6 +257,7 @@ class IntegrationReader:"""Pass `team_id` or `team_ids`, not both. An empty `team_ids` is False."""return self._matching(team_id, external_id, where, team_ids).exists()+ @instrumented("list_by_external_id", rows="returned")def list_by_external_id(self, external_id: str, *, team_ids: Iterable[int] | None = None) -> builtins.list[IntegrationRecord]:@@ -263,6 +270,7 @@ class IntegrationReader:rows = rows.filter(team_id__in=list(team_ids))return list(rows.order_by("created_at", "id"))+ @instrumented("get_by_external_id")def get_by_external_id(self, external_id: str, id: int) -> IntegrationRecord:"""The row with this id and this external id, in any team.@@ -275,19 +283,23 @@ class IntegrationReader:except (Integration.DoesNotExist, ValueError, TypeError) as error:raise IntegrationNotFound(f"No integration {id!r} with external id {external_id!r} in {self!r}") from error+ @instrumented("count_by_external_id")def count_by_external_id(self, external_id: str) -> int:"""The number of rows of every team with this external id."""return self._cross_team_rows("count_by_external_id").filter(integration_id=external_id).count()+ @instrumented("exists_by_external_id")def exists_by_external_id(self, external_id: str) -> bool:"""True when a row of any team has this external id."""return self._cross_team_rows("exists_by_external_id").filter(integration_id=external_id).exists()+ @instrumented("usages")def usages(self, team_id: int, id: int) -> builtins.list[IntegrationUsage]:"""The PostHog objects of the team that use the row. See `usages.py`."""- row = self.get(team_id, id)+ row = self._row(team_id, id)return integration_usages(team_id, row.id)+ @instrumented("delete")def delete(self, team_id: int, id: int, *, ignore_usages: bool = False) -> None:"""Delete the row. Raises `IntegrationInUse` when other objects use it, unless `ignore_usages`."""row = self._row(team_id, id)@@ -297,6 +309,7 @@ class IntegrationReader:raise IntegrationInUse(usages)row.delete()+ @instrumented("delete_by_external_id", rows="written")def delete_by_external_id(self, external_id: str) -> int:"""Delete the rows of every team with this external id, for example after an app is uninstalled.@@ -388,6 +401,7 @@ class AuthAccessor(IntegrationReader):def _in_scope(self, row: Integration) -> bool:return self._team_rows(row.team_id).filter(id=row.id).exists()+ @instrumented("create")def create(self,team_id: int,@@ -419,6 +433,7 @@ class AuthAccessor(IntegrationReader):f"Team {team_id} already has a {self._kind!r} integration with external id {external_id!r}") from error+ @instrumented("upsert")def upsert(self,team_id: int,@@ -460,6 +475,7 @@ class AuthAccessor(IntegrationReader):row.save(update_fields=update_fields)return row, False+ @instrumented("update")def update(self,team_id: int,@@ -487,6 +503,7 @@ class AuthAccessor(IntegrationReader):row.save(update_fields=update_fields)return row+ @instrumented("update_by_external_id", rows="written")def update_by_external_id(self, external_id: str, patch: IntegrationPatch) -> int:"""Apply `patch` to the rows of every team with this external id. Returns the number of rows that changed.@@ -592,6 +609,7 @@ class OAuthAccessor(AuthAccessor):def __init__(self, *, kind: str, source_rows: SourceRows = SourceRows.EXCLUDE) -> None:super().__init__(kind=kind, auth_type=AuthType.OAUTH, source_rows=source_rows)+ @instrumented("refresh_access_token")def refresh_access_token(self, team_id: int, id: int, *, stale_access_token: str | None = None) -> IntegrationRecord:@@ -613,8 +631,15 @@ class OAuthAccessor(AuthAccessor):row = self._row(team_id, id, lock=True)oauth = OauthIntegration(row)stale = stale_access_token is not None and row.access_token == stale_access_token- if stale or oauth.access_token_expired():+ if not (stale or oauth.access_token_expired()):+ record_token_refresh(self._kind, "fresh")+ return row+ try:oauth.refresh_access_token()+ except Exception:+ record_token_refresh(self._kind, "error")+ raise+ record_token_refresh(self._kind, "error" if row.errors else "stale_token" if stale else "refreshed")return rowasync def arefresh_access_token(diff --git a/posthog/integrations/observability.py b/posthog/integrations/observability.pynew file mode 100644index 000000000000..5716bfc3a663--- /dev/null+++ b/posthog/integrations/observability.py@@ -0,0 +1,173 @@+"""Prometheus metrics of the integrations SDK. See "Metrics" in README.md.++The accessor operations use the `instrumented` decorator, so one place counts every operation. The sync+and async forms count the same operation, because each async form calls its sync form. No label holds a+team id, an integration id, an external id or a config value.++Metrics must never break an SDK call. Every step that reads a label or writes a sample catches its+errors and logs them.+"""++import time+import functools+from collections.abc import Callable+from typing import TYPE_CHECKING, Literal, ParamSpec, TypeVar++import structlog+from prometheus_client import Counter, Histogram++from posthog.dataclasses import frozen+from posthog.integrations.contracts import (+ IntegrationConflict,+ IntegrationInUse,+ IntegrationNotFound,+ UnsupportedAuthType,+ UnsupportedIntegrationFilter,+)+from posthog.integrations.enums import AuthType++if TYPE_CHECKING:+ from posthog.integrations.accessors import IntegrationReader++logger = structlog.get_logger(__name__)++P = ParamSpec("P")+R = TypeVar("R")++RowsMeasure = Literal["none", "returned", "written"]+TokenRefreshResult = Literal["fresh", "refreshed", "stale_token", "error"]++MULTI_KIND = "multi"+ANY_AUTH_TYPE = "any"++_OPERATIONS = Counter(+ "integrations_sdk_operations_total",+ "Number of integrations SDK operations, by accessor, kind, auth type and outcome.",+ labelnames=["operation", "accessor", "kind", "auth_type", "outcome"],+)+_DURATION = Histogram(+ "integrations_sdk_operation_duration_seconds",+ "Time of one integrations SDK operation.",+ labelnames=["operation", "accessor", "outcome"],+ buckets=(0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5),+)+_ROWS_RETURNED = Histogram(+ "integrations_sdk_rows_returned",+ "Number of rows that a list-shaped integrations SDK read returned.",+ labelnames=["operation", "accessor"],+ buckets=(0, 1, 2, 5, 10, 50, 100, 500, 1000),+)+_ROWS_WRITTEN = Counter(+ "integrations_sdk_rows_written_total",+ "Number of rows that a cross-team integrations SDK write changed or deleted.",+ labelnames=["operation", "kind"],+)+_TOKEN_REFRESH = Counter(+ "integrations_sdk_token_refresh_total",+ "Number of integrations SDK access token refresh calls, by result.",+ labelnames=["kind", "result"],+)++_OUTCOMES: tuple[tuple[type[Exception], str], ...] = (+ (IntegrationNotFound, "not_found"),+ (IntegrationConflict, "conflict"),+ (IntegrationInUse, "in_use"),+ (UnsupportedIntegrationFilter, "unsupported_filter"),+ (UnsupportedAuthType, "unsupported_auth_type"),+)+++def outcome_of(error: Exception) -> str:+ for error_type, outcome in _OUTCOMES:+ if isinstance(error, error_type):+ return outcome+ return "error"+++@frozen+class AccessorLabels:+ accessor: str+ kind: str+ auth_type: str+++def accessor_labels(reader: "IntegrationReader") -> AccessorLabels:+ """The `accessor`, `kind` and `auth_type` labels of a reader, from its scope."""+ kinds = reader._kinds+ if reader._source_rows.value == "only":+ accessor = "source"+ elif kinds is None:+ accessor = "any"+ elif len(kinds) == 1:+ accessor = "named"+ else:+ accessor = "kinds"+ kind = next(iter(kinds)) if kinds is not None and len(kinds) == 1 else MULTI_KIND+ auth_type = reader._auth_type.value if isinstance(reader._auth_type, AuthType) else ANY_AUTH_TYPE+ return AccessorLabels(accessor=accessor, kind=kind, auth_type=auth_type)+++def _row_count(result: object) -> int | None:+ if isinstance(result, bool):+ return None+ if isinstance(result, int):+ return result+ try:+ return len(result) # type: ignore[arg-type]+ except TypeError:+ return None+++def _record(+ reader: "IntegrationReader",+ operation: str,+ started: float,+ outcome: str,+ rows: RowsMeasure,+ result: object,+) -> None:+ try:+ labels = accessor_labels(reader)+ accessor, kind, auth_type = labels.accessor, labels.kind, labels.auth_type+ _OPERATIONS.labels(operation, accessor, kind, auth_type, outcome).inc()+ _DURATION.labels(operation, accessor, outcome).observe(time.perf_counter() - started)+ if outcome != "ok" or rows == "none":+ return+ count = _row_count(result)+ if count is None:+ return+ if rows == "returned":+ _ROWS_RETURNED.labels(operation, accessor).observe(count)+ else:+ _ROWS_WRITTEN.labels(operation, kind).inc(count)+ except Exception:+ logger.warning("integrations_sdk_metrics_failed", operation=operation, exc_info=True)+++def instrumented(operation: str, *, rows: RowsMeasure = "none") -> Callable[[Callable[P, R]], Callable[P, R]]:+ """Count and time an accessor method. `rows` also records the size of the result: how many rows a+ read returned, or how many rows a cross-team write changed."""++ def decorator(method: Callable[P, R]) -> Callable[P, R]:+ @functools.wraps(method)+ def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:+ started = time.perf_counter()+ reader = args[0]+ try:+ result = method(*args, **kwargs)+ except Exception as error:+ _record(reader, operation, started, outcome_of(error), rows, None) # type: ignore[arg-type]+ raise+ _record(reader, operation, started, "ok", rows, result) # type: ignore[arg-type]+ return result++ return wrapper++ return decorator+++def record_token_refresh(kind: str, result: TokenRefreshResult) -> None:+ try:+ _TOKEN_REFRESH.labels(kind, result).inc()+ except Exception:+ logger.warning("integrations_sdk_metrics_failed", operation="refresh_access_token", exc_info=True)diff --git a/posthog/integrations/test/test_catalogue.py b/posthog/integrations/test/test_catalogue.pyindex 50211284d466..4cbd0316bed8 100644--- a/posthog/integrations/test/test_catalogue.py+++ b/posthog/integrations/test/test_catalogue.py@@ -146,6 +146,7 @@ SDK_FILES = ("enums.py","kinds_generated.py","manager.py",+ "observability.py","source_kinds.py","usages.py",)diff --git a/posthog/integrations/test/test_observability.py b/posthog/integrations/test/test_observability.pynew file mode 100644index 000000000000..24348f22fca9--- /dev/null+++ b/posthog/integrations/test/test_observability.py@@ -0,0 +1,279 @@+import time+from collections.abc import Callable+from typing import Any++from posthog.test.base import BaseTest+from unittest.mock import patch++from asgiref.sync import async_to_sync+from parameterized import parameterized+from prometheus_client import REGISTRY++from posthog.integrations import IntegrationManager, IntegrationPatch, observability, register_source_kinds, usages+from posthog.integrations.accessors import IntegrationReader, OAuthAccessor+from posthog.integrations.contracts import IntegrationInUse, IntegrationNotFound, IntegrationUsage+from posthog.integrations.enums import AuthType+from posthog.models import Team+from posthog.models.integration import ANY, Integration+from posthog.models.integration.oauth import OauthIntegration++OPERATIONS = "integrations_sdk_operations_total"+DURATION_COUNT = "integrations_sdk_operation_duration_seconds_count"+ROWS_RETURNED_SUM = "integrations_sdk_rows_returned_sum"+ROWS_WRITTEN = "integrations_sdk_rows_written_total"+TOKEN_REFRESH = "integrations_sdk_token_refresh_total"+++def _sample(name: str, **labels: str) -> float:+ return REGISTRY.get_sample_value(name, labels) or 0.0+++def _counts(operation: str, accessor: str, kind: str, auth_type: str, outcome: str) -> float:+ return _sample(OPERATIONS, operation=operation, accessor=accessor, kind=kind, auth_type=auth_type, outcome=outcome)+++class TestIntegrationsSdkMetrics(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ register_source_kinds({"ably": AuthType.TOKEN})+ self.github = IntegrationManager.github.oauth.create(self.team.id, external_id="gh-1", config={})+ self.team_id = self.team.id+ self.github_id = self.github.id++ @parameterized.expand(+ [+ (+ "get_ok",+ "get",+ "named",+ "github",+ "oauth",+ "ok",+ lambda s: IntegrationManager.github.oauth.get(s.team_id, s.github_id),+ ),+ (+ "get_not_found",+ "get",+ "named",+ "github",+ "oauth",+ "not_found",+ lambda s: IntegrationManager.github.oauth.get(s.team_id, 0),+ ),+ (+ "get_any_kind",+ "get",+ "any",+ "multi",+ "any",+ "ok",+ lambda s: IntegrationManager.any.get(s.team_id, s.github_id),+ ),+ (+ "list_kind_any_auth",+ "list",+ "named",+ "github",+ "any",+ "ok",+ lambda s: IntegrationManager.github.any.list(s.team_id),+ ),+ (+ "find_kinds",+ "find",+ "kinds",+ "multi",+ "any",+ "ok",+ lambda s: IntegrationManager.kinds("github", "slack").find(s.team_id),+ ),+ (+ "count_by_external_id",+ "count_by_external_id",+ "named",+ "github",+ "oauth",+ "ok",+ lambda s: IntegrationManager.github.oauth.count_by_external_id("gh-1"),+ ),+ (+ "any_cross_team_error",+ "list_by_external_id",+ "any",+ "multi",+ "any",+ "error",+ lambda s: IntegrationManager.any.list_by_external_id("gh-1"),+ ),+ (+ "create_conflict",+ "create",+ "named",+ "github",+ "oauth",+ "conflict",+ lambda s: IntegrationManager.github.oauth.create(s.team_id, external_id="gh-1", config={}),+ ),+ (+ "upsert_ok",+ "upsert",+ "named",+ "github",+ "oauth",+ "ok",+ lambda s: IntegrationManager.github.oauth.upsert(s.team_id, "gh-2", config={}),+ ),+ (+ "update_ok",+ "update",+ "named",+ "github",+ "oauth",+ "ok",+ lambda s: IntegrationManager.github.oauth.update(s.team_id, s.github_id, config_patch={"a": 1}),+ ),+ (+ "update_not_found",+ "update",+ "named",+ "github",+ "oauth",+ "not_found",+ lambda s: IntegrationManager.github.oauth.update(s.team_id, 0, config_patch={"a": 1}),+ ),+ (+ "delete_ok",+ "delete",+ "named",+ "github",+ "oauth",+ "ok",+ lambda s: IntegrationManager.github.oauth.delete(s.team_id, s.github_id),+ ),+ (+ "source_get_not_found",+ "get",+ "source",+ "ably",+ "any",+ "not_found",+ lambda s: IntegrationManager.source("ably").any.get(s.team_id, s.github_id),+ ),+ ]+ )+ def test_operation_is_counted_with_its_outcome(+ self,+ _name: str,+ operation: str,+ accessor: str,+ kind: str,+ auth_type: str,+ outcome: str,+ call: Callable[["TestIntegrationsSdkMetrics"], Any],+ ) -> None:+ before = _counts(operation, accessor, kind, auth_type, outcome)+ before_duration = _sample(DURATION_COUNT, operation=operation, accessor=accessor, outcome=outcome)++ try:+ call(self)+ except Exception:+ assert outcome != "ok"++ assert _counts(operation, accessor, kind, auth_type, outcome) == before + 1+ assert _sample(DURATION_COUNT, operation=operation, accessor=accessor, outcome=outcome) == before_duration + 1++ def test_delete_in_use_is_counted(self) -> None:+ before = _counts("delete", "named", "github", "oauth", "in_use")++ with (+ patch.object(usages, "_registered", []),+ patch.object(+ usages,+ "_product_providers",+ [lambda team_id, integration_id: [IntegrationUsage(type="workflow", id="1", name="Flow")]],+ ),+ ):+ with self.assertRaises(IntegrationInUse):+ IntegrationManager.github.oauth.delete(self.team_id, self.github_id)++ assert _counts("delete", "named", "github", "oauth", "in_use") == before + 1++ def test_usages_counts_one_operation(self) -> None:+ before_get = _counts("get", "named", "github", "oauth", "ok")+ before_usages = _counts("usages", "named", "github", "oauth", "ok")++ IntegrationManager.github.oauth.usages(self.team_id, self.github_id)++ assert _counts("get", "named", "github", "oauth", "ok") == before_get+ assert _counts("usages", "named", "github", "oauth", "ok") == before_usages + 1++ def test_rows_returned_and_written_are_recorded(self) -> None:+ IntegrationManager.github.oauth.create(self.team_id, external_id="gh-2", config={})+ other_team = Team.objects.create(organization=self.organization, name="Other")+ IntegrationManager.github.oauth.create(other_team.id, external_id="gh-1", config={})+ returned_before = _sample(ROWS_RETURNED_SUM, operation="list", accessor="named")+ written_before = _sample(ROWS_WRITTEN, operation="update_by_external_id", kind="github")++ listed = IntegrationManager.github.oauth.list(self.team_id)+ changed = IntegrationManager.github.oauth.update_by_external_id("gh-1", IntegrationPatch(config_patch={"a": 1}))++ assert len(listed) == 2+ assert changed == 2+ assert _sample(ROWS_RETURNED_SUM, operation="list", accessor="named") == returned_before + 2+ assert _sample(ROWS_WRITTEN, operation="update_by_external_id", kind="github") == written_before + 2++ def test_async_form_counts_the_same_operation(self) -> None:+ before = _counts("get", "named", "github", "oauth", "not_found")++ with patch.object(IntegrationReader, "_row", side_effect=IntegrationNotFound("missing")):+ with self.assertRaises(IntegrationNotFound):+ async_to_sync(IntegrationManager.github.oauth.aget)(self.team_id, self.github_id)++ assert _counts("get", "named", "github", "oauth", "not_found") == before + 1++ @parameterized.expand(+ [+ ("label_error", lambda: patch.object(observability, "accessor_labels", side_effect=RuntimeError("x"))),+ ("sample_error", lambda: patch.object(observability._OPERATIONS, "labels", side_effect=RuntimeError("x"))),+ ]+ )+ def test_broken_metrics_do_not_break_the_call(self, _name: str, broken_metrics: Callable[[], Any]) -> None:+ with broken_metrics():+ assert IntegrationManager.github.oauth.get(self.team_id, self.github_id).id == self.github_id+ with self.assertRaises(IntegrationNotFound):+ IntegrationManager.github.oauth.get(self.team_id, 0)++ @parameterized.expand(+ [+ ("fresh", 10, None, False, "", "fresh"),+ ("expired", 800, None, False, "", "refreshed"),+ ("stale_token", 10, "old", True, "", "stale_token"),+ ("failed_refresh_sets_errors", 800, None, True, "TOKEN_REFRESH_FAILED", "error"),+ ("raising_refresh", 800, None, True, "raise", "error"),+ ]+ )+ def test_token_refresh_result(+ self, _name: str, refreshed_ago: int, stale: str | None, calls_refresh: bool, errors: str, result: str+ ) -> None:+ row = Integration.objects._with_auth(ANY).create(+ team=self.team,+ kind="resend",+ integration_id="resend-1",+ config={"expires_in": 900, "refreshed_at": int(time.time()) - refreshed_ago},+ sensitive_config={"access_token": "old", "refresh_token": "r1"},+ )+ before = _sample(TOKEN_REFRESH, kind="resend", result=result)++ def refresh(oauth: OauthIntegration) -> None:+ if errors == "raise":+ raise RuntimeError("provider down")+ oauth.integration.errors = errors++ with patch.object(OauthIntegration, "refresh_access_token", autospec=True, side_effect=refresh) as mocked:+ try:+ OAuthAccessor(kind="resend").refresh_access_token(self.team_id, row.id, stale_access_token=stale)+ except RuntimeError:+ assert errors == "raise"++ assert mocked.call_count == (1 if calls_refresh or refreshed_ago > 600 else 0)+ assert _sample(TOKEN_REFRESH, kind="resend", result=result) == before + 1