:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(rubygems): add 5 tables for 4 missing rubygems endpoints (#128407)
Tom Owers5c6202547bdf
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex a13324911c79..4f70b0ce405e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10467,14 +10467,14 @@ Note: The v4 Core API exposes no download/listener analytics endpoint at all, so## Rubygems — gaps-Today (2): `gems`, `versions`+Today (7): `gems`, `versions`, `owners`, `reverse_dependencies`, `latest_gems`, `just_updated_gems`, `timeframe_versions`Diffed against: <https://guides.rubygems.org/rubygems-org-api/>-- [ ] `gems/{name}/owners.json` — membership table of gem maintainers, the only ownership join available (high)-- [ ] `gems/{name}/reverse_dependencies.json` — downstream adoption of a gem, the main ecosystem-impact question (medium)-- [ ] `activity/just_updated.json and activity/latest.json` — feed of newly released and newly updated gems for ecosystem tracking (medium)-- [ ] `timeframe_versions.json` — all versions published in a time window, an ecosystem-wide release feed with a natural incremental cursor (medium)+- [x] `gems/{name}/owners.json` — membership table of gem maintainers, the only ownership join available (high) → `owners` (full refresh, per configured gem)+- [x] `gems/{name}/reverse_dependencies.json` — downstream adoption of a gem, the main ecosystem-impact question (medium) → `reverse_dependencies` (full refresh, per configured gem)+- [x] `activity/just_updated.json and activity/latest.json` — feed of newly released and newly updated gems for ecosystem tracking (medium) → `just_updated_gems`, `latest_gems` (full refresh, ecosystem-wide)+- [x] `timeframe_versions.json` — all versions published in a time window, an ecosystem-wide release feed with a natural incremental cursor (medium) → `timeframe_versions` (incremental on `created_at`, 7-day first sync)- [ ] `owners/{handle}/gems.json` — inverse of the owners join, lists every gem for a maintainer (low)- [ ] `profiles/{handle}.json` — lookup resolving owner handles to profile metadata (low)diff --git a/products/warehouse_sources/sources/rubygems/canonical_descriptions.py b/products/warehouse_sources/sources/rubygems/canonical_descriptions.pyindex 072a768e329a..781e1e459e47 100644--- a/products/warehouse_sources/sources/rubygems/canonical_descriptions.py+++ b/products/warehouse_sources/sources/rubygems/canonical_descriptions.py@@ -55,4 +55,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"spec_sha": "SHA-256 checksum of this version's gemspec.",},},+ "owners": {+ "description": "Every user who can push to each configured gem, one row per gem and owner.",+ "docs_url": _API_DOCS,+ "columns": {+ "gem_name": "Gem the owner belongs to (injected by the connector).",+ "id": "RubyGems.org user id of the owner.",+ "handle": "RubyGems.org handle of the owner.",+ "email": "Email address of the owner, present only when the user made it public.",+ "role": "Role of the user on the gem: owner or maintainer.",+ },+ },+ "reverse_dependencies": {+ "description": "Gems whose latest version depends on a configured gem, one row per gem and dependent.",+ "docs_url": _API_DOCS,+ "columns": {+ "gem_name": "Configured gem that is depended on (injected by the connector).",+ "dependent_name": "Name of a gem whose latest version depends on `gem_name`, at runtime or in development.",+ },+ },+ "latest_gems": {+ "description": "The 50 gems most recently added to RubyGems.org for the first time, across all gems.",+ "docs_url": _API_DOCS,+ "columns": {+ "name": "Gem name, as registered on RubyGems.org.",+ "version": "Latest published version number.",+ "version_created_at": "Timestamp the latest version was published.",+ "downloads": "Total cumulative downloads of the gem across all versions and platforms.",+ },+ },+ "just_updated_gems": {+ "description": "The 50 most recently pushed gem versions on RubyGems.org, across all gems.",+ "docs_url": _API_DOCS,+ "columns": {+ "name": "Gem name, as registered on RubyGems.org.",+ "version": "Version number that was pushed.",+ "platform": "Build platform of the pushed version (e.g. ruby, java, x86_64-linux).",+ "version_created_at": "Timestamp the version was pushed.",+ },+ },+ "timeframe_versions": {+ "description": "Every gem version pushed to RubyGems.org, across all gems, one row per version and platform.",+ "docs_url": _API_DOCS,+ "columns": {+ "name": "Gem the version belongs to.",+ "number": "Version number (e.g. 1.2.3).",+ "platform": "Build platform this version was published for (e.g. ruby, java, x86_64-linux).",+ "created_at": "Timestamp the version was published; used as the incremental cursor and partition key.",+ "downloads_count": "Cumulative downloads of this specific version.",+ "prerelease": "Whether this version is a prerelease.",+ "sha": "SHA-256 checksum of this version's `.gem` file.",+ },+ },}diff --git a/products/warehouse_sources/sources/rubygems/rubygems.py b/products/warehouse_sources/sources/rubygems/rubygems.pyindex 81a06b4a1b8f..1c875a722c41 100644--- a/products/warehouse_sources/sources/rubygems/rubygems.py+++ b/products/warehouse_sources/sources/rubygems/rubygems.py@@ -1,14 +1,20 @@import re+import datetime as dtfrom collections.abc import Callable, Iterator-from typing import Any-from urllib.parse import quote+from typing import Any, Optional+from urllib.parse import quote, urlencodeimport requestsfrom structlog.types import FilteringBoundLoggerfrom tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.rubygems.settings import RUBYGEMS_ENDPOINTS, RubyGemsEndpointConfig-from products.warehouse_sources.sources.sdk import SourceResponse, make_tracked_session, retry_with_deadline+from products.warehouse_sources.sources.sdk import (+ SourceResponse,+ make_tracked_session,+ parse_datetime_value,+ retry_with_deadline,+)RUBYGEMS_BASE_URL = "https://rubygems.org/api/v1"@@ -26,6 +32,13 @@ _HEADERS = {"Accept": "application/json"}# this, so the exact value only caps the per-yield list size.MAX_ROWS_PER_BATCH = 5000+# `timeframe_versions` rejects a `from`..`to` range longer than 7 days and pages 30 versions at a time.+TIMEFRAME_MAX_WINDOW = dt.timedelta(days=7)+TIMEFRAME_PAGE_SIZE = 30+# A first sync, or one with no recorded history start, reads this far back. The feed covers every gem+# on RubyGems.org, so the whole history would take hundreds of thousands of requests.+TIMEFRAME_HISTORY_LOOKBACK = dt.timedelta(days=7)+class RubyGemsRetryableError(Exception):pass@@ -70,13 +83,43 @@ def _versions_url(gem: str) -> str:return f"{RUBYGEMS_BASE_URL}/versions/{quote(gem, safe='')}.json"+def _owners_url(gem: str) -> str:+ return f"{RUBYGEMS_BASE_URL}/gems/{quote(gem, safe='')}/owners.json"+++def _reverse_dependencies_url(gem: str) -> str:+ return f"{RUBYGEMS_BASE_URL}/gems/{quote(gem, safe='')}/reverse_dependencies.json"+++def _timeframe_versions_url(start: dt.datetime, end: dt.datetime, page: int) -> str:+ params = {"from": _iso8601(start), "to": _iso8601(end), "page": page}+ return f"{RUBYGEMS_BASE_URL}/timeframe_versions.json?{urlencode(params)}"+++def _iso8601(value: dt.datetime) -> str:+ return value.astimezone(dt.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")+++_PER_GEM_URLS: dict[str, Callable[[str], str]] = {+ "gems": _gem_url,+ "versions": _versions_url,+ "owners": _owners_url,+ "reverse_dependencies": _reverse_dependencies_url,+}++_FEED_URLS: dict[str, str] = {+ "latest_gems": f"{RUBYGEMS_BASE_URL}/activity/latest.json",+ "just_updated_gems": f"{RUBYGEMS_BASE_URL}/activity/just_updated.json",+}++@retry_with_deadline(retry=retry_if_exception_type((RubyGemsRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),wait=wait_exponential_jitter(initial=1, max=30),reraise=True,)-def _fetch(session: requests.Session, url: str, gem: str, logger: FilteringBoundLogger) -> Any | None:+def _fetch(session: requests.Session, url: str, label: str, logger: FilteringBoundLogger) -> Any | None:"""Fetch a single JSON document.Returns ``None`` for a 404 (gem not found) so a typo'd or unpublished gem is skipped rather@@ -86,7 +129,7 @@ def _fetch(session: requests.Session, url: str, gem: str, logger: FilteringBoundresponse = session.get(url, headers=_HEADERS, timeout=REQUEST_TIMEOUT_SECONDS)if response.status_code == 404:- logger.warning(f"RubyGems: gem {gem!r} not found, skipping")+ logger.warning(f"RubyGems: {label!r} not found, skipping")return Noneif response.status_code == 429 or response.status_code >= 500:@@ -126,9 +169,34 @@ def _version_rows(gem: str, versions: list[Any]) -> Iterator[dict[str, Any]]:yield row+def _owner_rows(gem: str, owners: list[Any]) -> Iterator[dict[str, Any]]:+ """One row per owner of a gem, stamped with the parent `gem_name`."""+ for owner in owners:+ if not isinstance(owner, dict) or owner.get("id") is None:+ continue+ row = dict(owner)+ row["gem_name"] = gem+ yield row+++def _reverse_dependency_rows(gem: str, dependents: list[Any]) -> Iterator[dict[str, Any]]:+ """One row per dependent gem. The API returns bare gem names, so wrap each in a row."""+ for dependent in dependents:+ if not isinstance(dependent, str) or not dependent:+ continue+ yield {"gem_name": gem, "dependent_name": dependent}+++def _keyed_rows(rows: list[Any], primary_keys: list[str]) -> list[dict[str, Any]]:+ # A row missing a primary key component would merge with a null key, so drop it.+ return [row for row in rows if isinstance(row, dict) and all(row.get(key) for key in primary_keys)]++_ROW_BUILDERS: dict[str, Callable[[str, Any], Iterator[dict[str, Any]]]] = {"gems": _gem_rows,"versions": _version_rows,+ "owners": _owner_rows,+ "reverse_dependencies": _reverse_dependency_rows,}@@ -166,16 +234,13 @@ def get_rows(session = make_tracked_session()for gem in gems:+ document = _fetch(session, _PER_GEM_URLS[endpoint](gem), gem, logger)+ if document is None:+ continueif endpoint == "gems":- document = _fetch(session, _gem_url(gem), gem, logger)- if document is None:- continuesource_data: Any = documentelse:- versions = _fetch(session, _versions_url(gem), gem, logger)- if versions is None:- continue- source_data = versions if isinstance(versions, list) else []+ source_data = document if isinstance(document, list) else []build_rows = _ROW_BUILDERS[endpoint]# Stream the builder into bounded chunks: a gem with a very large version history is never@@ -191,13 +256,64 @@ def get_rows(yield chunk+def get_feed_rows(endpoint: str, logger: FilteringBoundLogger) -> Iterator[list[dict[str, Any]]]:+ document = _fetch(make_tracked_session(), _FEED_URLS[endpoint], endpoint, logger)+ rows = _keyed_rows(document if isinstance(document, list) else [], RUBYGEMS_ENDPOINTS[endpoint].primary_keys)+ if rows:+ yield rows+++def get_timeframe_version_rows(+ start: dt.datetime,+ logger: FilteringBoundLogger,+) -> Iterator[list[dict[str, Any]]]:+ """Walk 7-day windows from ``start`` to now, paging each window until a short page."""+ session = make_tracked_session()+ end = dt.datetime.now(dt.UTC)+ primary_keys = RUBYGEMS_ENDPOINTS["timeframe_versions"].primary_keys++ window_start = start+ while window_start < end:+ window_end = min(window_start + TIMEFRAME_MAX_WINDOW, end)+ page = 1+ while True:+ versions = _fetch(+ session, _timeframe_versions_url(window_start, window_end, page), "timeframe_versions", logger+ )+ if not isinstance(versions, list) or not versions:+ break+ rows = _keyed_rows(versions, primary_keys)+ if rows:+ yield rows+ if len(versions) < TIMEFRAME_PAGE_SIZE:+ break+ page += 1+ window_start = window_end+++def timeframe_start(+ should_use_incremental_field: bool,+ last_value: Any,+ history_start: Optional[dt.datetime],+) -> dt.datetime:+ if should_use_incremental_field and last_value is not None:+ watermark = parse_datetime_value(last_value)+ if watermark is not None:+ return watermark+ if history_start is not None:+ return history_start+ return dt.datetime.now(dt.UTC) - TIMEFRAME_HISTORY_LOOKBACK++def rubygems_source(endpoint: str,gems_raw: str | None,logger: FilteringBoundLogger,+ should_use_incremental_field: bool = False,+ db_incremental_field_last_value: Any = None,+ history_start: Optional[dt.datetime] = None,) -> SourceResponse:config: RubyGemsEndpointConfig = RUBYGEMS_ENDPOINTS[endpoint]- gems = parse_gems(gems_raw)partition_kwargs: dict[str, Any] = {}if config.partition_key is not None:@@ -209,9 +325,27 @@ def rubygems_source("partition_keys": [config.partition_key],}+ if endpoint == "timeframe_versions":+ start = timeframe_start(should_use_incremental_field, db_incremental_field_last_value, history_start)+ return SourceResponse(+ name=endpoint,+ items=lambda: get_timeframe_version_rows(start=start, logger=logger),+ primary_keys=config.primary_keys,+ # The API returns each window oldest first in practice, but does not document an order.+ sort_mode=None,+ **partition_kwargs,+ )++ gems = parse_gems(gems_raw) if config.per_gem else []++ def items() -> Iterator[list[dict[str, Any]]]:+ if config.per_gem:+ return get_rows(endpoint=endpoint, gems=gems, logger=logger)+ return get_feed_rows(endpoint=endpoint, logger=logger)+return SourceResponse(name=endpoint,- items=lambda: get_rows(endpoint=endpoint, gems=gems, logger=logger),+ items=items,primary_keys=config.primary_keys,# No server-side ordering to rely on; rows are grouped per gem as fetched.sort_mode="asc",diff --git a/products/warehouse_sources/sources/rubygems/settings.py b/products/warehouse_sources/sources/rubygems/settings.pyindex 2ebc653a5293..1ccfbaaf78be 100644--- a/products/warehouse_sources/sources/rubygems/settings.py+++ b/products/warehouse_sources/sources/rubygems/settings.py@@ -25,6 +25,9 @@ class RubyGemsEndpointConfig:partition_key: Optional[str] = Noneshould_sync_default: bool = Truedescription: Optional[str] = None+ # Ecosystem-wide feeds ignore the configured gem list and make the same requests for every source.+ per_gem: bool = True+ supports_incremental: bool = FalseRUBYGEMS_ENDPOINTS: dict[str, RubyGemsEndpointConfig] = {@@ -46,6 +49,47 @@ RUBYGEMS_ENDPOINTS: dict[str, RubyGemsEndpointConfig] = {description="Every published version of each configured gem, including per-version download ""counts, from GET /api/v1/versions/{name}.json.",),+ "owners": RubyGemsEndpointConfig(+ name="owners",+ # Owner ids are global user ids, and one user can own many configured gems.+ primary_keys=["gem_name", "id"],+ description="Every owner and maintainer of each configured gem, from GET /api/v1/gems/{name}/owners.json.",+ ),+ "reverse_dependencies": RubyGemsEndpointConfig(+ name="reverse_dependencies",+ primary_keys=["gem_name", "dependent_name"],+ description="Every gem whose latest version depends on a configured gem, from "+ "GET /api/v1/gems/{name}/reverse_dependencies.json.",+ ),+ "latest_gems": RubyGemsEndpointConfig(+ name="latest_gems",+ primary_keys=["name"],+ should_sync_default=False,+ per_gem=False,+ description="The 50 gems most recently added to RubyGems.org, across the whole ecosystem, from "+ "GET /api/v1/activity/latest.json.",+ ),+ "just_updated_gems": RubyGemsEndpointConfig(+ name="just_updated_gems",+ # The feed has one row per pushed version, so a gem that ships several platforms in a row+ # appears once per platform.+ primary_keys=["name", "version", "platform"],+ should_sync_default=False,+ per_gem=False,+ description="The 50 most recently pushed gem versions, across the whole ecosystem, from "+ "GET /api/v1/activity/just_updated.json.",+ ),+ "timeframe_versions": RubyGemsEndpointConfig(+ name="timeframe_versions",+ primary_keys=["name", "number", "platform"],+ incremental_fields=list(_CREATED_AT_INCREMENTAL_FIELDS),+ partition_key="created_at",+ should_sync_default=False,+ per_gem=False,+ supports_incremental=True,+ description="Every gem version pushed to RubyGems.org, across the whole ecosystem, from "+ "GET /api/v1/timeframe_versions.json. A first sync reads the last 7 days.",+ ),}ENDPOINTS = tuple(RUBYGEMS_ENDPOINTS.keys())diff --git a/products/warehouse_sources/sources/rubygems/source.py b/products/warehouse_sources/sources/rubygems/source.pyindex a2303100ee67..3fb187edff6b 100644--- a/products/warehouse_sources/sources/rubygems/source.py+++ b/products/warehouse_sources/sources/rubygems/source.py@@ -1,7 +1,9 @@+import datetimefrom typing import Optional, castfrom products.warehouse_sources.sources.rubygems._config import RubygemsSourceConfigfrom products.warehouse_sources.sources.rubygems.rubygems import (+ TIMEFRAME_HISTORY_LOOKBACK,rubygems_source,validate_credentials as validate_rubygems_credentials,)@@ -52,7 +54,9 @@ rspecdevise```-Each sync fetches the current metadata and full version history for every configured gem. RubyGems.org has no server-side "changed since" filter, so all tables sync as a full refresh.""",+Each sync fetches the current metadata, version history, owners and reverse dependencies for every configured gem. These tables sync as a full refresh, because RubyGems.org has no server-side "changed since" filter for them.++The `latest_gems`, `just_updated_gems` and `timeframe_versions` tables cover all of RubyGems.org, not only your configured gems. `timeframe_versions` syncs incrementally, and its first sync reads the last 7 days.""",iconPath="/static/services/rubygems.png",docsUrl="https://posthog.com/docs/cdp/sources/rubygems",fields=cast(@@ -71,6 +75,12 @@ Each sync fetches the current metadata and full version history for every config),)+ def history_lookback_for_schema(+ self, schema_name: str, config: RubygemsSourceConfig | None = None+ ) -> datetime.timedelta | None:+ # Only the ecosystem-wide release feed is bounded; the per-gem tables return every row.+ return TIMEFRAME_HISTORY_LOOKBACK if schema_name == "timeframe_versions" else None+def get_canonical_descriptions(self) -> CanonicalDescriptions:from products.warehouse_sources.sources.rubygems.canonical_descriptions import (CANONICAL_DESCRIPTIONS, # noqa: PLC0415@@ -95,9 +105,9 @@ Each sync fetches the current metadata and full version history for every configschemas = [SourceSchema(name=endpoint,- # RubyGems.org exposes no server-side timestamp filter, so no stream is truly- # incremental; re-fetching would only duplicate immutable history, so append is off too.- supports_incremental=False,+ # Only `timeframe_versions` takes a server-side time filter; re-fetching the per-gem+ # streams would only duplicate immutable history, so append is off too.+ supports_incremental=RUBYGEMS_ENDPOINTS[endpoint].supports_incremental,supports_append=False,incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),should_sync_default=RUBYGEMS_ENDPOINTS[endpoint].should_sync_default,@@ -124,4 +134,7 @@ Each sync fetches the current metadata and full version history for every configendpoint=inputs.schema_name,gems_raw=config.gems,logger=inputs.logger,+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,+ history_start=inputs.history_start,)diff --git a/products/warehouse_sources/sources/rubygems/tests/test_rubygems.py b/products/warehouse_sources/sources/rubygems/tests/test_rubygems.pyindex 8ff87b8c8ff6..d1c3a77cb029 100644--- a/products/warehouse_sources/sources/rubygems/tests/test_rubygems.py+++ b/products/warehouse_sources/sources/rubygems/tests/test_rubygems.py@@ -1,16 +1,20 @@import json+import datetime as dtfrom typing import Any, Optionalimport pytest+import time_machinefrom unittest import mockimport requestsimport structlog+from parameterized import parameterizedfrom products.warehouse_sources.sources.rubygems._config import RubygemsSourceConfigfrom products.warehouse_sources.sources.rubygems.rubygems import (MAX_GEMS,RUBYGEMS_BASE_URL,+ TIMEFRAME_PAGE_SIZE,RubyGemsRetryableError,_fetch,parse_gems,@@ -169,3 +173,141 @@ class TestRubyGemsSource:def test_invalid_gems_raise(self):with pytest.raises(ValueError):rubygems_source("gems", "", structlog.get_logger())+++class TestPerGemTables:+ @parameterized.expand(+ [+ (+ "owners",+ "owners.json",+ [{"id": 1, "handle": "qrush", "role": "owner"}, {"handle": "no-id"}, "not-a-dict"],+ [{"id": 1, "handle": "qrush", "role": "owner", "gem_name": "rails"}],+ ),+ (+ "reverse_dependencies",+ "reverse_dependencies.json",+ ["jeweler", "", None, "soup"],+ [+ {"gem_name": "rails", "dependent_name": "jeweler"},+ {"gem_name": "rails", "dependent_name": "soup"},+ ],+ ),+ ]+ )+ def test_rows_are_stamped_with_the_gem_and_404_gems_are_skipped(self, endpoint, suffix, body, expected_rows):+ driver = SourceDriver(RubygemsSource(), RubygemsSourceConfig(gems="rails\nnope"))++ result = driver.run(endpoint, [ScriptedResponse(json=body), ScriptedResponse(status=404, json={})])++ assert result.raised is None+ assert result.paths == [f"/api/v1/gems/rails/{suffix}", f"/api/v1/gems/nope/{suffix}"]+ assert result.rows == expected_rows+++class TestFeedTables:+ @parameterized.expand(+ [+ (+ "latest_gems",+ "/api/v1/activity/latest.json",+ [{"name": "brand-new", "version": "0.1.0"}, {"version": "no-name"}],+ [{"name": "brand-new", "version": "0.1.0"}],+ ),+ (+ "just_updated_gems",+ "/api/v1/activity/just_updated.json",+ [+ {"name": "nokogiri", "version": "1.18.0", "platform": "ruby"},+ {"name": "nokogiri", "version": "1.18.0", "platform": "java"},+ {"name": "nokogiri", "version": "1.18.0", "platform": None},+ ],+ [+ {"name": "nokogiri", "version": "1.18.0", "platform": "ruby"},+ {"name": "nokogiri", "version": "1.18.0", "platform": "java"},+ ],+ ),+ ]+ )+ def test_one_request_ignores_configured_gems_and_drops_rows_without_a_key(self, endpoint, path, body, expected):+ driver = SourceDriver(RubygemsSource(), RubygemsSourceConfig(gems="rails\nrspec"))++ result = driver.run(endpoint, [ScriptedResponse(json=body)])++ assert result.raised is None+ assert result.paths == [path]+ assert result.rows == expected+++def _timeframe_page(count: int, created_at: str = "2026-11-01T00:00:00.000Z") -> list[dict[str, Any]]:+ return [{"name": f"gem{i}", "number": "1.0.0", "platform": "ruby", "created_at": created_at} for i in range(count)]+++class TestTimeframeVersions:+ @pytest.fixture(autouse=True)+ def _frozen_now(self):+ with time_machine.travel(dt.datetime(2026, 11, 3, 12, tzinfo=dt.UTC), tick=False):+ yield++ @parameterized.expand(+ [+ ("incremental_watermark", True, "2026-11-01T06:30:00.000Z", None, "2026-11-01T06:30:00Z"),+ (+ "full_refresh_uses_history_start",+ False,+ "2026-11-01T06:30:00Z",+ dt.datetime(2026, 10, 30, tzinfo=dt.UTC),+ "2026-10-30T00:00:00Z",+ ),+ (+ "incremental_without_watermark_uses_history_start",+ True,+ None,+ dt.datetime(2026, 10, 30, tzinfo=dt.UTC),+ "2026-10-30T00:00:00Z",+ ),+ ("no_bound_falls_back_to_lookback", False, None, None, "2026-10-27T12:00:00Z"),+ ]+ )+ def test_start_of_the_read(self, _name, incremental, last_value, history_start, expected_from):+ driver = SourceDriver(RubygemsSource(), RubygemsSourceConfig(gems="rails"))++ result = driver.run(+ "timeframe_versions",+ [ScriptedResponse(json=[])],+ should_use_incremental_field=incremental,+ incremental_field="created_at",+ db_incremental_field_last_value=last_value,+ history_start=history_start,+ )++ assert result.raised is None+ assert result.params("from") == [expected_from]+ assert result.params("to") == ["2026-11-03T12:00:00Z"]++ def test_walks_pages_and_seven_day_windows_to_now(self):+ driver = SourceDriver(RubygemsSource(), RubygemsSourceConfig(gems="rails"))++ result = driver.run(+ "timeframe_versions",+ [+ # First window: a full page, then a short page ends it.+ ScriptedResponse(json=_timeframe_page(TIMEFRAME_PAGE_SIZE)),+ ScriptedResponse(json=[*_timeframe_page(2), {"name": "no-platform", "number": "1.0.0"}]),+ # Second window: an empty first page ends it, and the walk stops at now.+ ScriptedResponse(json=[]),+ ],+ should_use_incremental_field=True,+ incremental_field="created_at",+ db_incremental_field_last_value="2026-10-27T00:00:00Z",+ )++ assert result.raised is None+ assert result.paths == ["/api/v1/timeframe_versions.json"] * 3+ assert result.params("from") == ["2026-10-27T00:00:00Z", "2026-10-27T00:00:00Z", "2026-11-03T00:00:00Z"]+ assert result.params("to") == ["2026-11-03T00:00:00Z", "2026-11-03T00:00:00Z", "2026-11-03T12:00:00Z"]+ assert result.params("page") == ["1", "2", "1"]+ assert len(result.rows) == TIMEFRAME_PAGE_SIZE + 2+ assert result.response is not None+ assert result.response.sort_mode is None+ assert result.response.partition_keys == ["created_at"]