Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(rubygems): add 5 tables for 4 missing rubygems endpoints (#128407)

Tom Owers5c6202547bdf
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index a13324911c79..4f70b0ce405e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10467,14 +10467,14 @@ Note: The v4 Core API exposes no download/listener analytics endpoint at all, so
## Rubygems — gaps
-Today (2): `gems`, `versions`
+Today (7): `gems`, `versions`, `owners`, `reverse_dependencies`, `latest_gems`, `just_updated_gems`, `timeframe_versions`
Diffed against: <https://guides.rubygems.org/rubygems-org-api/>
-- [ ] `gems/{name}/owners.json` — membership table of gem maintainers, the only ownership join available (high)
-- [ ] `gems/{name}/reverse_dependencies.json` — downstream adoption of a gem, the main ecosystem-impact question (medium)
-- [ ] `activity/just_updated.json and activity/latest.json` — feed of newly released and newly updated gems for ecosystem tracking (medium)
-- [ ] `timeframe_versions.json` — all versions published in a time window, an ecosystem-wide release feed with a natural incremental cursor (medium)
+- [x] `gems/{name}/owners.json` — membership table of gem maintainers, the only ownership join available (high) → `owners` (full refresh, per configured gem)
+- [x] `gems/{name}/reverse_dependencies.json` — downstream adoption of a gem, the main ecosystem-impact question (medium) → `reverse_dependencies` (full refresh, per configured gem)
+- [x] `activity/just_updated.json and activity/latest.json` — feed of newly released and newly updated gems for ecosystem tracking (medium) → `just_updated_gems`, `latest_gems` (full refresh, ecosystem-wide)
+- [x] `timeframe_versions.json` — all versions published in a time window, an ecosystem-wide release feed with a natural incremental cursor (medium) → `timeframe_versions` (incremental on `created_at`, 7-day first sync)
- [ ] `owners/{handle}/gems.json` — inverse of the owners join, lists every gem for a maintainer (low)
- [ ] `profiles/{handle}.json` — lookup resolving owner handles to profile metadata (low)
diff --git a/products/warehouse_sources/sources/rubygems/canonical_descriptions.py b/products/warehouse_sources/sources/rubygems/canonical_descriptions.py
index 072a768e329a..781e1e459e47 100644
--- a/products/warehouse_sources/sources/rubygems/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/rubygems/canonical_descriptions.py
@@ -55,4 +55,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"spec_sha": "SHA-256 checksum of this version's gemspec.",
},
},
+ "owners": {
+ "description": "Every user who can push to each configured gem, one row per gem and owner.",
+ "docs_url": _API_DOCS,
+ "columns": {
+ "gem_name": "Gem the owner belongs to (injected by the connector).",
+ "id": "RubyGems.org user id of the owner.",
+ "handle": "RubyGems.org handle of the owner.",
+ "email": "Email address of the owner, present only when the user made it public.",
+ "role": "Role of the user on the gem: owner or maintainer.",
+ },
+ },
+ "reverse_dependencies": {
+ "description": "Gems whose latest version depends on a configured gem, one row per gem and dependent.",
+ "docs_url": _API_DOCS,
+ "columns": {
+ "gem_name": "Configured gem that is depended on (injected by the connector).",
+ "dependent_name": "Name of a gem whose latest version depends on `gem_name`, at runtime or in development.",
+ },
+ },
+ "latest_gems": {
+ "description": "The 50 gems most recently added to RubyGems.org for the first time, across all gems.",
+ "docs_url": _API_DOCS,
+ "columns": {
+ "name": "Gem name, as registered on RubyGems.org.",
+ "version": "Latest published version number.",
+ "version_created_at": "Timestamp the latest version was published.",
+ "downloads": "Total cumulative downloads of the gem across all versions and platforms.",
+ },
+ },
+ "just_updated_gems": {
+ "description": "The 50 most recently pushed gem versions on RubyGems.org, across all gems.",
+ "docs_url": _API_DOCS,
+ "columns": {
+ "name": "Gem name, as registered on RubyGems.org.",
+ "version": "Version number that was pushed.",
+ "platform": "Build platform of the pushed version (e.g. ruby, java, x86_64-linux).",
+ "version_created_at": "Timestamp the version was pushed.",
+ },
+ },
+ "timeframe_versions": {
+ "description": "Every gem version pushed to RubyGems.org, across all gems, one row per version and platform.",
+ "docs_url": _API_DOCS,
+ "columns": {
+ "name": "Gem the version belongs to.",
+ "number": "Version number (e.g. 1.2.3).",
+ "platform": "Build platform this version was published for (e.g. ruby, java, x86_64-linux).",
+ "created_at": "Timestamp the version was published; used as the incremental cursor and partition key.",
+ "downloads_count": "Cumulative downloads of this specific version.",
+ "prerelease": "Whether this version is a prerelease.",
+ "sha": "SHA-256 checksum of this version's `.gem` file.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/rubygems/rubygems.py b/products/warehouse_sources/sources/rubygems/rubygems.py
index 81a06b4a1b8f..1c875a722c41 100644
--- a/products/warehouse_sources/sources/rubygems/rubygems.py
+++ b/products/warehouse_sources/sources/rubygems/rubygems.py
@@ -1,14 +1,20 @@
import re
+import datetime as dt
from collections.abc import Callable, Iterator
-from typing import Any
-from urllib.parse import quote
+from typing import Any, Optional
+from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.rubygems.settings import RUBYGEMS_ENDPOINTS, RubyGemsEndpointConfig
-from products.warehouse_sources.sources.sdk import SourceResponse, make_tracked_session, retry_with_deadline
+from products.warehouse_sources.sources.sdk import (
+ SourceResponse,
+ make_tracked_session,
+ parse_datetime_value,
+ retry_with_deadline,
+)
RUBYGEMS_BASE_URL = "https://rubygems.org/api/v1"
@@ -26,6 +32,13 @@ _HEADERS = {"Accept": "application/json"}
# this, so the exact value only caps the per-yield list size.
MAX_ROWS_PER_BATCH = 5000
+# `timeframe_versions` rejects a `from`..`to` range longer than 7 days and pages 30 versions at a time.
+TIMEFRAME_MAX_WINDOW = dt.timedelta(days=7)
+TIMEFRAME_PAGE_SIZE = 30
+# A first sync, or one with no recorded history start, reads this far back. The feed covers every gem
+# on RubyGems.org, so the whole history would take hundreds of thousands of requests.
+TIMEFRAME_HISTORY_LOOKBACK = dt.timedelta(days=7)
+
class RubyGemsRetryableError(Exception):
pass
@@ -70,13 +83,43 @@ def _versions_url(gem: str) -> str:
return f"{RUBYGEMS_BASE_URL}/versions/{quote(gem, safe='')}.json"
+def _owners_url(gem: str) -> str:
+ return f"{RUBYGEMS_BASE_URL}/gems/{quote(gem, safe='')}/owners.json"
+
+
+def _reverse_dependencies_url(gem: str) -> str:
+ return f"{RUBYGEMS_BASE_URL}/gems/{quote(gem, safe='')}/reverse_dependencies.json"
+
+
+def _timeframe_versions_url(start: dt.datetime, end: dt.datetime, page: int) -> str:
+ params = {"from": _iso8601(start), "to": _iso8601(end), "page": page}
+ return f"{RUBYGEMS_BASE_URL}/timeframe_versions.json?{urlencode(params)}"
+
+
+def _iso8601(value: dt.datetime) -> str:
+ return value.astimezone(dt.UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+_PER_GEM_URLS: dict[str, Callable[[str], str]] = {
+ "gems": _gem_url,
+ "versions": _versions_url,
+ "owners": _owners_url,
+ "reverse_dependencies": _reverse_dependencies_url,
+}
+
+_FEED_URLS: dict[str, str] = {
+ "latest_gems": f"{RUBYGEMS_BASE_URL}/activity/latest.json",
+ "just_updated_gems": f"{RUBYGEMS_BASE_URL}/activity/just_updated.json",
+}
+
+
@retry_with_deadline(
retry=retry_if_exception_type((RubyGemsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
reraise=True,
)
-def _fetch(session: requests.Session, url: str, gem: str, logger: FilteringBoundLogger) -> Any | None:
+def _fetch(session: requests.Session, url: str, label: str, logger: FilteringBoundLogger) -> Any | None:
"""Fetch a single JSON document.
Returns ``None`` for a 404 (gem not found) so a typo'd or unpublished gem is skipped rather
@@ -86,7 +129,7 @@ def _fetch(session: requests.Session, url: str, gem: str, logger: FilteringBound
response = session.get(url, headers=_HEADERS, timeout=REQUEST_TIMEOUT_SECONDS)
if response.status_code == 404:
- logger.warning(f"RubyGems: gem {gem!r} not found, skipping")
+ logger.warning(f"RubyGems: {label!r} not found, skipping")
return None
if response.status_code == 429 or response.status_code >= 500:
@@ -126,9 +169,34 @@ def _version_rows(gem: str, versions: list[Any]) -> Iterator[dict[str, Any]]:
yield row
+def _owner_rows(gem: str, owners: list[Any]) -> Iterator[dict[str, Any]]:
+ """One row per owner of a gem, stamped with the parent `gem_name`."""
+ for owner in owners:
+ if not isinstance(owner, dict) or owner.get("id") is None:
+ continue
+ row = dict(owner)
+ row["gem_name"] = gem
+ yield row
+
+
+def _reverse_dependency_rows(gem: str, dependents: list[Any]) -> Iterator[dict[str, Any]]:
+ """One row per dependent gem. The API returns bare gem names, so wrap each in a row."""
+ for dependent in dependents:
+ if not isinstance(dependent, str) or not dependent:
+ continue
+ yield {"gem_name": gem, "dependent_name": dependent}
+
+
+def _keyed_rows(rows: list[Any], primary_keys: list[str]) -> list[dict[str, Any]]:
+ # A row missing a primary key component would merge with a null key, so drop it.
+ return [row for row in rows if isinstance(row, dict) and all(row.get(key) for key in primary_keys)]
+
+
_ROW_BUILDERS: dict[str, Callable[[str, Any], Iterator[dict[str, Any]]]] = {
"gems": _gem_rows,
"versions": _version_rows,
+ "owners": _owner_rows,
+ "reverse_dependencies": _reverse_dependency_rows,
}
@@ -166,16 +234,13 @@ def get_rows(
session = make_tracked_session()
for gem in gems:
+ document = _fetch(session, _PER_GEM_URLS[endpoint](gem), gem, logger)
+ if document is None:
+ continue
if endpoint == "gems":
- document = _fetch(session, _gem_url(gem), gem, logger)
- if document is None:
- continue
source_data: Any = document
else:
- versions = _fetch(session, _versions_url(gem), gem, logger)
- if versions is None:
- continue
- source_data = versions if isinstance(versions, list) else []
+ source_data = document if isinstance(document, list) else []
build_rows = _ROW_BUILDERS[endpoint]
# Stream the builder into bounded chunks: a gem with a very large version history is never
@@ -191,13 +256,64 @@ def get_rows(
yield chunk
+def get_feed_rows(endpoint: str, logger: FilteringBoundLogger) -> Iterator[list[dict[str, Any]]]:
+ document = _fetch(make_tracked_session(), _FEED_URLS[endpoint], endpoint, logger)
+ rows = _keyed_rows(document if isinstance(document, list) else [], RUBYGEMS_ENDPOINTS[endpoint].primary_keys)
+ if rows:
+ yield rows
+
+
+def get_timeframe_version_rows(
+ start: dt.datetime,
+ logger: FilteringBoundLogger,
+) -> Iterator[list[dict[str, Any]]]:
+ """Walk 7-day windows from ``start`` to now, paging each window until a short page."""
+ session = make_tracked_session()
+ end = dt.datetime.now(dt.UTC)
+ primary_keys = RUBYGEMS_ENDPOINTS["timeframe_versions"].primary_keys
+
+ window_start = start
+ while window_start < end:
+ window_end = min(window_start + TIMEFRAME_MAX_WINDOW, end)
+ page = 1
+ while True:
+ versions = _fetch(
+ session, _timeframe_versions_url(window_start, window_end, page), "timeframe_versions", logger
+ )
+ if not isinstance(versions, list) or not versions:
+ break
+ rows = _keyed_rows(versions, primary_keys)
+ if rows:
+ yield rows
+ if len(versions) < TIMEFRAME_PAGE_SIZE:
+ break
+ page += 1
+ window_start = window_end
+
+
+def timeframe_start(
+ should_use_incremental_field: bool,
+ last_value: Any,
+ history_start: Optional[dt.datetime],
+) -> dt.datetime:
+ if should_use_incremental_field and last_value is not None:
+ watermark = parse_datetime_value(last_value)
+ if watermark is not None:
+ return watermark
+ if history_start is not None:
+ return history_start
+ return dt.datetime.now(dt.UTC) - TIMEFRAME_HISTORY_LOOKBACK
+
+
def rubygems_source(
endpoint: str,
gems_raw: str | None,
logger: FilteringBoundLogger,
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
+ history_start: Optional[dt.datetime] = None,
) -> SourceResponse:
config: RubyGemsEndpointConfig = RUBYGEMS_ENDPOINTS[endpoint]
- gems = parse_gems(gems_raw)
partition_kwargs: dict[str, Any] = {}
if config.partition_key is not None:
@@ -209,9 +325,27 @@ def rubygems_source(
"partition_keys": [config.partition_key],
}
+ if endpoint == "timeframe_versions":
+ start = timeframe_start(should_use_incremental_field, db_incremental_field_last_value, history_start)
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: get_timeframe_version_rows(start=start, logger=logger),
+ primary_keys=config.primary_keys,
+ # The API returns each window oldest first in practice, but does not document an order.
+ sort_mode=None,
+ **partition_kwargs,
+ )
+
+ gems = parse_gems(gems_raw) if config.per_gem else []
+
+ def items() -> Iterator[list[dict[str, Any]]]:
+ if config.per_gem:
+ return get_rows(endpoint=endpoint, gems=gems, logger=logger)
+ return get_feed_rows(endpoint=endpoint, logger=logger)
+
return SourceResponse(
name=endpoint,
- items=lambda: get_rows(endpoint=endpoint, gems=gems, logger=logger),
+ items=items,
primary_keys=config.primary_keys,
# No server-side ordering to rely on; rows are grouped per gem as fetched.
sort_mode="asc",
diff --git a/products/warehouse_sources/sources/rubygems/settings.py b/products/warehouse_sources/sources/rubygems/settings.py
index 2ebc653a5293..1ccfbaaf78be 100644
--- a/products/warehouse_sources/sources/rubygems/settings.py
+++ b/products/warehouse_sources/sources/rubygems/settings.py
@@ -25,6 +25,9 @@ class RubyGemsEndpointConfig:
partition_key: Optional[str] = None
should_sync_default: bool = True
description: Optional[str] = None
+ # Ecosystem-wide feeds ignore the configured gem list and make the same requests for every source.
+ per_gem: bool = True
+ supports_incremental: bool = False
RUBYGEMS_ENDPOINTS: dict[str, RubyGemsEndpointConfig] = {
@@ -46,6 +49,47 @@ RUBYGEMS_ENDPOINTS: dict[str, RubyGemsEndpointConfig] = {
description="Every published version of each configured gem, including per-version download "
"counts, from GET /api/v1/versions/{name}.json.",
),
+ "owners": RubyGemsEndpointConfig(
+ name="owners",
+ # Owner ids are global user ids, and one user can own many configured gems.
+ primary_keys=["gem_name", "id"],
+ description="Every owner and maintainer of each configured gem, from GET /api/v1/gems/{name}/owners.json.",
+ ),
+ "reverse_dependencies": RubyGemsEndpointConfig(
+ name="reverse_dependencies",
+ primary_keys=["gem_name", "dependent_name"],
+ description="Every gem whose latest version depends on a configured gem, from "
+ "GET /api/v1/gems/{name}/reverse_dependencies.json.",
+ ),
+ "latest_gems": RubyGemsEndpointConfig(
+ name="latest_gems",
+ primary_keys=["name"],
+ should_sync_default=False,
+ per_gem=False,
+ description="The 50 gems most recently added to RubyGems.org, across the whole ecosystem, from "
+ "GET /api/v1/activity/latest.json.",
+ ),
+ "just_updated_gems": RubyGemsEndpointConfig(
+ name="just_updated_gems",
+ # The feed has one row per pushed version, so a gem that ships several platforms in a row
+ # appears once per platform.
+ primary_keys=["name", "version", "platform"],
+ should_sync_default=False,
+ per_gem=False,
+ description="The 50 most recently pushed gem versions, across the whole ecosystem, from "
+ "GET /api/v1/activity/just_updated.json.",
+ ),
+ "timeframe_versions": RubyGemsEndpointConfig(
+ name="timeframe_versions",
+ primary_keys=["name", "number", "platform"],
+ incremental_fields=list(_CREATED_AT_INCREMENTAL_FIELDS),
+ partition_key="created_at",
+ should_sync_default=False,
+ per_gem=False,
+ supports_incremental=True,
+ description="Every gem version pushed to RubyGems.org, across the whole ecosystem, from "
+ "GET /api/v1/timeframe_versions.json. A first sync reads the last 7 days.",
+ ),
}
ENDPOINTS = tuple(RUBYGEMS_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/rubygems/source.py b/products/warehouse_sources/sources/rubygems/source.py
index a2303100ee67..3fb187edff6b 100644
--- a/products/warehouse_sources/sources/rubygems/source.py
+++ b/products/warehouse_sources/sources/rubygems/source.py
@@ -1,7 +1,9 @@
+import datetime
from typing import Optional, cast
from products.warehouse_sources.sources.rubygems._config import RubygemsSourceConfig
from products.warehouse_sources.sources.rubygems.rubygems import (
+ TIMEFRAME_HISTORY_LOOKBACK,
rubygems_source,
validate_credentials as validate_rubygems_credentials,
)
@@ -52,7 +54,9 @@ rspec
devise
```
-Each sync fetches the current metadata and full version history for every configured gem. RubyGems.org has no server-side "changed since" filter, so all tables sync as a full refresh.""",
+Each sync fetches the current metadata, version history, owners and reverse dependencies for every configured gem. These tables sync as a full refresh, because RubyGems.org has no server-side "changed since" filter for them.
+
+The `latest_gems`, `just_updated_gems` and `timeframe_versions` tables cover all of RubyGems.org, not only your configured gems. `timeframe_versions` syncs incrementally, and its first sync reads the last 7 days.""",
iconPath="/static/services/rubygems.png",
docsUrl="https://posthog.com/docs/cdp/sources/rubygems",
fields=cast(
@@ -71,6 +75,12 @@ Each sync fetches the current metadata and full version history for every config
),
)
+ def history_lookback_for_schema(
+ self, schema_name: str, config: RubygemsSourceConfig | None = None
+ ) -> datetime.timedelta | None:
+ # Only the ecosystem-wide release feed is bounded; the per-gem tables return every row.
+ return TIMEFRAME_HISTORY_LOOKBACK if schema_name == "timeframe_versions" else None
+
def get_canonical_descriptions(self) -> CanonicalDescriptions:
from products.warehouse_sources.sources.rubygems.canonical_descriptions import (
CANONICAL_DESCRIPTIONS, # noqa: PLC0415
@@ -95,9 +105,9 @@ Each sync fetches the current metadata and full version history for every config
schemas = [
SourceSchema(
name=endpoint,
- # RubyGems.org exposes no server-side timestamp filter, so no stream is truly
- # incremental; re-fetching would only duplicate immutable history, so append is off too.
- supports_incremental=False,
+ # Only `timeframe_versions` takes a server-side time filter; re-fetching the per-gem
+ # streams would only duplicate immutable history, so append is off too.
+ supports_incremental=RUBYGEMS_ENDPOINTS[endpoint].supports_incremental,
supports_append=False,
incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
should_sync_default=RUBYGEMS_ENDPOINTS[endpoint].should_sync_default,
@@ -124,4 +134,7 @@ Each sync fetches the current metadata and full version history for every config
endpoint=inputs.schema_name,
gems_raw=config.gems,
logger=inputs.logger,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
+ history_start=inputs.history_start,
)
diff --git a/products/warehouse_sources/sources/rubygems/tests/test_rubygems.py b/products/warehouse_sources/sources/rubygems/tests/test_rubygems.py
index 8ff87b8c8ff6..d1c3a77cb029 100644
--- a/products/warehouse_sources/sources/rubygems/tests/test_rubygems.py
+++ b/products/warehouse_sources/sources/rubygems/tests/test_rubygems.py
@@ -1,16 +1,20 @@
import json
+import datetime as dt
from typing import Any, Optional
import pytest
+import time_machine
from unittest import mock
import requests
import structlog
+from parameterized import parameterized
from products.warehouse_sources.sources.rubygems._config import RubygemsSourceConfig
from products.warehouse_sources.sources.rubygems.rubygems import (
MAX_GEMS,
RUBYGEMS_BASE_URL,
+ TIMEFRAME_PAGE_SIZE,
RubyGemsRetryableError,
_fetch,
parse_gems,
@@ -169,3 +173,141 @@ class TestRubyGemsSource:
def test_invalid_gems_raise(self):
with pytest.raises(ValueError):
rubygems_source("gems", "", structlog.get_logger())
+
+
+class TestPerGemTables:
+ @parameterized.expand(
+ [
+ (
+ "owners",
+ "owners.json",
+ [{"id": 1, "handle": "qrush", "role": "owner"}, {"handle": "no-id"}, "not-a-dict"],
+ [{"id": 1, "handle": "qrush", "role": "owner", "gem_name": "rails"}],
+ ),
+ (
+ "reverse_dependencies",
+ "reverse_dependencies.json",
+ ["jeweler", "", None, "soup"],
+ [
+ {"gem_name": "rails", "dependent_name": "jeweler"},
+ {"gem_name": "rails", "dependent_name": "soup"},
+ ],
+ ),
+ ]
+ )
+ def test_rows_are_stamped_with_the_gem_and_404_gems_are_skipped(self, endpoint, suffix, body, expected_rows):
+ driver = SourceDriver(RubygemsSource(), RubygemsSourceConfig(gems="rails\nnope"))
+
+ result = driver.run(endpoint, [ScriptedResponse(json=body), ScriptedResponse(status=404, json={})])
+
+ assert result.raised is None
+ assert result.paths == [f"/api/v1/gems/rails/{suffix}", f"/api/v1/gems/nope/{suffix}"]
+ assert result.rows == expected_rows
+
+
+class TestFeedTables:
+ @parameterized.expand(
+ [
+ (
+ "latest_gems",
+ "/api/v1/activity/latest.json",
+ [{"name": "brand-new", "version": "0.1.0"}, {"version": "no-name"}],
+ [{"name": "brand-new", "version": "0.1.0"}],
+ ),
+ (
+ "just_updated_gems",
+ "/api/v1/activity/just_updated.json",
+ [
+ {"name": "nokogiri", "version": "1.18.0", "platform": "ruby"},
+ {"name": "nokogiri", "version": "1.18.0", "platform": "java"},
+ {"name": "nokogiri", "version": "1.18.0", "platform": None},
+ ],
+ [
+ {"name": "nokogiri", "version": "1.18.0", "platform": "ruby"},
+ {"name": "nokogiri", "version": "1.18.0", "platform": "java"},
+ ],
+ ),
+ ]
+ )
+ def test_one_request_ignores_configured_gems_and_drops_rows_without_a_key(self, endpoint, path, body, expected):
+ driver = SourceDriver(RubygemsSource(), RubygemsSourceConfig(gems="rails\nrspec"))
+
+ result = driver.run(endpoint, [ScriptedResponse(json=body)])
+
+ assert result.raised is None
+ assert result.paths == [path]
+ assert result.rows == expected
+
+
+def _timeframe_page(count: int, created_at: str = "2026-11-01T00:00:00.000Z") -> list[dict[str, Any]]:
+ return [{"name": f"gem{i}", "number": "1.0.0", "platform": "ruby", "created_at": created_at} for i in range(count)]
+
+
+class TestTimeframeVersions:
+ @pytest.fixture(autouse=True)
+ def _frozen_now(self):
+ with time_machine.travel(dt.datetime(2026, 11, 3, 12, tzinfo=dt.UTC), tick=False):
+ yield
+
+ @parameterized.expand(
+ [
+ ("incremental_watermark", True, "2026-11-01T06:30:00.000Z", None, "2026-11-01T06:30:00Z"),
+ (
+ "full_refresh_uses_history_start",
+ False,
+ "2026-11-01T06:30:00Z",
+ dt.datetime(2026, 10, 30, tzinfo=dt.UTC),
+ "2026-10-30T00:00:00Z",
+ ),
+ (
+ "incremental_without_watermark_uses_history_start",
+ True,
+ None,
+ dt.datetime(2026, 10, 30, tzinfo=dt.UTC),
+ "2026-10-30T00:00:00Z",
+ ),
+ ("no_bound_falls_back_to_lookback", False, None, None, "2026-10-27T12:00:00Z"),
+ ]
+ )
+ def test_start_of_the_read(self, _name, incremental, last_value, history_start, expected_from):
+ driver = SourceDriver(RubygemsSource(), RubygemsSourceConfig(gems="rails"))
+
+ result = driver.run(
+ "timeframe_versions",
+ [ScriptedResponse(json=[])],
+ should_use_incremental_field=incremental,
+ incremental_field="created_at",
+ db_incremental_field_last_value=last_value,
+ history_start=history_start,
+ )
+
+ assert result.raised is None
+ assert result.params("from") == [expected_from]
+ assert result.params("to") == ["2026-11-03T12:00:00Z"]
+
+ def test_walks_pages_and_seven_day_windows_to_now(self):
+ driver = SourceDriver(RubygemsSource(), RubygemsSourceConfig(gems="rails"))
+
+ result = driver.run(
+ "timeframe_versions",
+ [
+ # First window: a full page, then a short page ends it.
+ ScriptedResponse(json=_timeframe_page(TIMEFRAME_PAGE_SIZE)),
+ ScriptedResponse(json=[*_timeframe_page(2), {"name": "no-platform", "number": "1.0.0"}]),
+ # Second window: an empty first page ends it, and the walk stops at now.
+ ScriptedResponse(json=[]),
+ ],
+ should_use_incremental_field=True,
+ incremental_field="created_at",
+ db_incremental_field_last_value="2026-10-27T00:00:00Z",
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/api/v1/timeframe_versions.json"] * 3
+ assert result.params("from") == ["2026-10-27T00:00:00Z", "2026-10-27T00:00:00Z", "2026-11-03T00:00:00Z"]
+ assert result.params("to") == ["2026-11-03T00:00:00Z", "2026-11-03T00:00:00Z", "2026-11-03T12:00:00Z"]
+ assert result.params("page") == ["1", "2", "1"]
+ assert len(result.rows) == TIMEFRAME_PAGE_SIZE + 2
+ assert result.response is not None
+ assert result.response.sort_mode is None
+ assert result.response.partition_keys == ["created_at"]