:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(google-sheets): read worksheet headers in one request during discovery (#126128)
Tom Owers6e2d40d5ccb1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/google_sheets.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/google_sheets.pyindex 164a571bb0b9..736f3567349c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/google_sheets.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/google_sheets.py@@ -2,8 +2,9 @@ import reimport timeimport randomfrom collections import Counter-from collections.abc import Callable, Collection+from collections.abc import Callable, Collection, Iteratorfrom typing import Any, Optional, TypeVar, cast+from urllib.parse import quoteimport gspreadimport requests@@ -11,7 +12,7 @@ from cachetools import Cache, TTLCache, cachedfrom google.auth import exceptions as google_auth_exceptionsfrom google.auth.transport.requests import AuthorizedSessionfrom google.oauth2 import service_account-from gspread.utils import numericise_all+from gspread.utils import absolute_range_name, fill_gaps, numericise_allfrom posthog.credentials_service.errors import CredentialsSecretErrorfrom posthog.dataclasses import frozen@@ -346,17 +347,21 @@ def get_schemas(config: GoogleSheetsSourceConfig) -> list[tuple[str, int]]:def get_worksheets(config: GoogleSheetsSourceConfig) -> list[DiscoveredWorksheet]:- return [_discovered_worksheet(worksheet) for worksheet in _list_worksheets(config, _SYNC_RETRY_BUDGET)]+ _, worksheets = _list_worksheets(config, _SYNC_RETRY_BUDGET)+ return [_discovered_worksheet(worksheet) for worksheet in worksheets]-def _list_worksheets(config: GoogleSheetsSourceConfig, budget: _RetryBudget) -> list[gspread.Worksheet]:+def _list_worksheets(+ config: GoogleSheetsSourceConfig, budget: _RetryBudget+) -> tuple[gspread.Spreadsheet, list[gspread.Worksheet]]:client = google_sheets_client()# `open_by_url` and `worksheets()` hit the Sheets API and so are subject to the same# transient quota (429) and 5xx server errors as `_get_worksheet` — retry them with backoff# rather than letting a transient blip fail the whole sync during schema discovery.- def execute() -> list[gspread.Worksheet]:- return _open_spreadsheet(client, config.spreadsheet_url).worksheets()+ def execute() -> tuple[gspread.Spreadsheet, list[gspread.Worksheet]]:+ spreadsheet = _open_spreadsheet(client, config.spreadsheet_url)+ return spreadsheet, spreadsheet.worksheets()return _retry_on_transient_api_error(execute, budget)@@ -378,22 +383,79 @@ class DiscoveredWorksheetSchema:def discover_worksheet_schemas(config: GoogleSheetsSourceConfig, names: Collection[str] | None = None) -> list[DiscoveredWorksheetSchema]:- """List the worksheets once and read each header row through the listed handles.-- The handles share one client and one token, so each worksheet costs one header read and- no further spreadsheet lookups."""- schemas: list[DiscoveredWorksheetSchema] = []- for handle in _list_worksheets(config, _DISCOVERY_RETRY_BUDGET):- worksheet = _discovered_worksheet(handle)- if names is not None and worksheet.name not in names:- continue- schemas.append(- DiscoveredWorksheetSchema(- worksheet=worksheet,- incremental_fields=_incremental_fields_from_header(handle, _DISCOVERY_RETRY_BUDGET),- )+ """List the worksheets once and read the header rows in as few batched requests as the URL allows.++ Discovery runs while the user waits in the source wizard, and every Sheets request spends a quota+ that all PostHog customers share. One read per worksheet lets a spreadsheet with many tabs outlast+ the request timeout."""+ spreadsheet, handles = _list_worksheets(config, _DISCOVERY_RETRY_BUDGET)+ selected = [handle for handle in handles if names is None or _discovered_worksheet(handle).name in names]+ header_rows = _read_header_rows(spreadsheet, selected, _DISCOVERY_RETRY_BUDGET)+ return [+ DiscoveredWorksheetSchema(+ worksheet=_discovered_worksheet(handle),+ incremental_fields=_incremental_fields_from_rows(rows),)- return schemas+ for handle, rows in zip(selected, header_rows, strict=True)+ ]+++def _is_unparseable_range_error(e: gspread.exceptions.APIError) -> bool:+ return e.code == 400 and "Unable to parse range" in str(e)+++# `values:batchGet` is a GET, so every range goes into the URL, and Google rejects a URL much past+# 16 KB. A spreadsheet with hundreds of long tab names needs several batches.+_MAX_BATCH_RANGES_URL_LENGTH = 8000+++def _header_range(worksheet: gspread.Worksheet) -> str:+ return absolute_range_name(worksheet.title, "1:2")+++def _header_read_batches(worksheets: list[gspread.Worksheet]) -> Iterator[list[gspread.Worksheet]]:+ batch: list[gspread.Worksheet] = []+ batch_length = 0+ for worksheet in worksheets:+ length = len("&ranges=") + len(quote(_header_range(worksheet), safe=""))+ if batch and batch_length + length > _MAX_BATCH_RANGES_URL_LENGTH:+ yield batch+ batch, batch_length = [], 0+ batch.append(worksheet)+ batch_length += length+ if batch:+ yield batch+++def _read_header_rows(+ spreadsheet: gspread.Spreadsheet, worksheets: list[gspread.Worksheet], budget: _RetryBudget+) -> list[list[list[Any]]]:+ rows: list[list[list[Any]]] = []+ for batch in _header_read_batches(worksheets):+ rows.extend(_read_header_rows_batch(spreadsheet, batch, budget))+ return rows+++def _read_header_rows_batch(+ spreadsheet: gspread.Spreadsheet, worksheets: list[gspread.Worksheet], budget: _RetryBudget+) -> list[list[list[Any]]]:+ ranges = [_header_range(worksheet) for worksheet in worksheets]+ try:+ response = _retry_on_transient_api_error(lambda: spreadsheet.values_batch_get(ranges), budget)+ except gspread.exceptions.APIError as e:+ if not _is_unparseable_range_error(e):+ raise+ # One unreadable worksheet fails the whole batch, so read each worksheet on its own to find it.+ return [_read_worksheet_header_rows(worksheet, budget) for worksheet in worksheets]+ return [_padded_rows(value_range.get("values", [[]])) for value_range in response.get("valueRanges", [])]+++def _padded_rows(rows: list[list[Any]]) -> list[list[Any]]:+ # Mirrors `Worksheet.get_all_values`, which pads short rows to the widest one.+ try:+ return fill_gaps(rows)+ except KeyError:+ return [[]]def _resolve_worksheet_id(@@ -415,9 +477,9 @@ def _resolve_worksheet_id(return selected_worksheet[0]-def _incremental_fields_from_header(worksheet: gspread.Worksheet, budget: _RetryBudget) -> list[IncrementalField]:+def _read_worksheet_header_rows(worksheet: gspread.Worksheet, budget: _RetryBudget) -> list[list[Any]]:try:- rows = _retry_on_transient_api_error(lambda: worksheet.get_all_values("1:2"), budget) # Get the first two rows+ return _retry_on_transient_api_error(lambda: worksheet.get_all_values("1:2"), budget) # Get the first two rowsexcept gspread.exceptions.APIError as e:# Google rejects the unbounded "1:2" row range with a 400 "Unable to parse range" for# some worksheets (e.g. empty sheets, or sheets resized to have no columns). This is@@ -426,10 +488,12 @@ def _incremental_fields_from_header(worksheet: gspread.Worksheet, budget: _Retry# incremental field — return [] rather than letting one odd sheet break schema discovery# for the entire spreadsheet. Other API errors (e.g. transient 5xx) still propagate so# they can be retried.- if e.code == 400 and "Unable to parse range" in str(e):+ if _is_unparseable_range_error(e):return []raise++def _incremental_fields_from_rows(rows: list[list[Any]]) -> list[IncrementalField]:if len(rows) > 1 and "id" in rows[0]:index_of_id = rows[0].index("id")value_of_id_col = rows[1][index_of_id]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/tests/test_google_sheets.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/tests/test_google_sheets.pyindex 00e8508b3210..ca2dfa1ee1d9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/tests/test_google_sheets.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_sheets/tests/test_google_sheets.py@@ -1,5 +1,6 @@from collections.abc import Iterablefrom typing import Any, cast+from urllib.parse import urlencodeimport pytestfrom unittest import mock@@ -538,17 +539,20 @@ def test_discover_worksheet_schemas_skips_unparseable_range():config = GoogleSheetsSourceConfig(spreadsheet_url="https://docs.google.com/spreadsheets/d/fake")handle = _worksheet_handle("csm_followups", 123)- handle.get_all_values.side_effect = _api_error(- 400, "Unable to parse range: 'csm_followups'!1:2", "INVALID_ARGUMENT"- )+ unparseable = _api_error(400, "Unable to parse range: 'csm_followups'!1:2", "INVALID_ARGUMENT")+ handle.get_all_values.side_effect = unparseable+ readable = _worksheet_handle("budget", 124)+ readable.get_all_values.return_value = [["id"], [1]]+ client = _sheets_client_listing(handle, readable)+ client.open_by_url.return_value.values_batch_get.side_effect = unparseablewith mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.google_sheets.google_sheets.google_sheets_client",- return_value=_sheets_client_listing(handle),+ return_value=client,):- [schema] = discover_worksheet_schemas(config)+ schemas = discover_worksheet_schemas(config)- assert schema.incremental_fields == []+ assert [len(schema.incremental_fields) for schema in schemas] == [0, 1]def test_google_sheets_source_skips_unparseable_header_range():@@ -620,7 +624,9 @@ def test_discover_worksheet_schemas_reraises_other_api_errors_after_a_short_retrconfig = GoogleSheetsSourceConfig(spreadsheet_url="https://docs.google.com/spreadsheets/d/fake")handle = _worksheet_handle("sheet1", 123)- handle.get_all_values.side_effect = _api_error(500, "Internal error encountered.", "INTERNAL")+ client = _sheets_client_listing(handle)+ batch_get = client.open_by_url.return_value.values_batch_get+ batch_get.side_effect = _api_error(500, "Internal error encountered.", "INTERNAL")with (mock.patch(@@ -628,13 +634,13 @@ def test_discover_worksheet_schemas_reraises_other_api_errors_after_a_short_retr) as mock_time,mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.google_sheets.google_sheets.google_sheets_client",- return_value=_sheets_client_listing(handle),+ return_value=client,),pytest.raises(gspread.exceptions.APIError),):discover_worksheet_schemas(config)- assert handle.get_all_values.call_count == 3+ assert batch_get.call_count == 3assert sum(call.args[0] for call in mock_time.sleep.call_args_list) < 10@@ -868,12 +874,31 @@ def test_google_sheets_source_resolves_the_stored_worksheet_id_before_the_name(assert mock_get_worksheet.call_args.args[1] == expected_worksheet_id+def test_discovery_splits_header_reads_to_keep_each_request_url_bounded():+ handles = [_worksheet_handle(f"{index:03d} " + "quarterly revenue by region " * 3, index) for index in range(300)]+ client = _sheets_client_listing(*handles)+ batch_get = client.open_by_url.return_value.values_batch_get+ batch_get.side_effect = lambda ranges: {"valueRanges": [{"values": [["text"]]} for _ in ranges]}++ with mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.google_sheets.google_sheets.google_sheets_client",+ return_value=client,+ ):+ config = GoogleSheetsSourceConfig(spreadsheet_url="https://docs.google.com/spreadsheets/d/fake")+ schemas = discover_worksheet_schemas(config)++ batches = [call.args[0] for call in batch_get.call_args_list]+ assert len(batches) > 1+ assert all(len(urlencode([("ranges", r) for r in batch])) <= 8000 for batch in batches)+ assert [schema.worksheet.worksheet_id for schema in schemas] == list(range(300))++def test_discovered_schemas_list_worksheets_once_and_carry_title_and_id():budget = _worksheet_handle("Budget 2025", 7)- budget.get_all_values.return_value = [["id"], [1]]notes = _worksheet_handle("Notes", 8)- notes.get_all_values.return_value = [["text"], ["hello"]]client = _sheets_client_listing(budget, notes)+ batch_get = client.open_by_url.return_value.values_batch_get+ batch_get.return_value = {"valueRanges": [{"values": [["id"], [1]]}, {"values": [["text"], ["hello"]]}]}with mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.google_sheets.google_sheets.google_sheets_client",@@ -885,8 +910,10 @@ def test_discovered_schemas_list_worksheets_once_and_carry_title_and_id():assert mock_client_factory.call_count == 1assert client.open_by_url.call_count == 1assert client.open_by_url.return_value.worksheets.call_count == 1- assert budget.get_all_values.call_count == 1- assert notes.get_all_values.call_count == 1+ assert batch_get.call_count == 1+ assert batch_get.call_args.args[0] == ["'Budget 2025'!1:2", "'Notes'!1:2"]+ assert budget.get_all_values.call_count == 0+ assert notes.get_all_values.call_count == 0assert [(s.name, s.label, s.schema_metadata, s.supports_incremental) for s in schemas] == [("budget_2025", "Budget 2025", {SCHEMA_RESOURCE_ID_METADATA_KEY: "7"}, True),("notes", "Notes", {SCHEMA_RESOURCE_ID_METADATA_KEY: "8"}, False),