:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from typing import Optional, cast
from products.warehouse_sources.sources.crossref._config import CrossrefSourceConfig
from products.warehouse_sources.sources.crossref.crossref import (
CrossrefResumeConfig,
crossref_source,
validate_credentials as validate_crossref_credentials,
)
from products.warehouse_sources.sources.crossref.settings import ENDPOINTS, INCREMENTAL_FIELDS
from products.warehouse_sources.sources.sdk import (
CanonicalDescriptions,
DataWarehouseSourceCategory,
ExternalDataSourceType,
FieldType,
ReleaseStatus,
ResumableSource,
ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceInputs,
SourceRegistry,
SourceResponse,
SourceSchema,
build_endpoint_schemas,
detect,
)
_SCOPE_REQUIRED_MESSAGE = (
"Set a member ID, funder ID, or journal ISSN before syncing Works. Crossref indexes over "
"160 million works, so an unscoped sync isn't practical — scope it to a publisher, funder, "
"or journal first."
)
@SourceRegistry.register
class CrossrefSource(ResumableSource[CrossrefSourceConfig, CrossrefResumeConfig]):
codebase_detection = detect(env=("^CROSSREF_",), pypi=("habanero", "crossrefapi"))
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
api_docs_url = "https://github.com/CrossRef/rest-api-doc"
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.CROSSREF
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
name=ExternalDataSourceType.CROSSREF,
category=DataWarehouseSourceCategory.ANALYTICS,
label="Crossref",
releaseStatus=ReleaseStatus.BETA,
caption="""Crossref's API is free and public — no API key needed. Add a contact email to get routed to Crossref's faster "polite pool".
The Works table covers Crossref's full DOI registry (160 million+ records), so set a member ID, funder ID, or journal ISSN below to scope which works sync. The Members, Funders, Types, Licenses, and Journals tables always sync in full.
""",
iconPath="/static/services/crossref.png",
docsUrl="https://posthog.com/docs/cdp/sources/crossref",
fields=cast(
list[FieldType],
[
SourceFieldInputConfig(
name="mailto",
label="Contact email (optional)",
type=SourceFieldInputConfigType.EMAIL,
required=False,
placeholder="[email protected]",
secret=False,
),
SourceFieldInputConfig(
name="member_id",
sync_setting=True,
label="Crossref member ID (optional)",
type=SourceFieldInputConfigType.TEXT,
required=False,
placeholder="301",
secret=False,
caption="Scopes the Works table to one publisher. Look up IDs at [api.crossref.org/members](https://api.crossref.org/members).",
),
SourceFieldInputConfig(
name="funder_id",
sync_setting=True,
label="Funder ID (optional)",
type=SourceFieldInputConfigType.TEXT,
required=False,
placeholder="100000001",
secret=False,
caption="Scopes the Works table to one funder from the [Open Funder Registry](https://api.crossref.org/funders).",
),
SourceFieldInputConfig(
name="issn",
sync_setting=True,
label="Journal ISSN (optional)",
type=SourceFieldInputConfigType.TEXT,
required=False,
placeholder="1932-6203",
secret=False,
caption="Scopes the Works table to one journal.",
),
],
),
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
from products.warehouse_sources.sources.crossref.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
def get_non_retryable_errors(self) -> dict[str, str | None]:
return {
"must be a positive integer": "Crossref rejected the member ID or funder ID as invalid. Check the value and try again.",
}
def get_schemas(
self,
config: CrossrefSourceConfig,
team_id: int,
with_counts: bool = False,
names: list[str] | None = None,
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
return build_endpoint_schemas(
ENDPOINTS.keys(),
INCREMENTAL_FIELDS,
names,
# Works rows get re-indexed/re-deposited over time, so append would accumulate one
# duplicate row per re-fetch of the same DOI; only merge (dedupe on DOI) is offered.
merge_only={"Works"},
descriptions={name: cfg.description for name, cfg in ENDPOINTS.items() if cfg.description},
)
def _has_scope(self, config: CrossrefSourceConfig) -> bool:
return bool(config.member_id or config.funder_id or config.issn)
def validate_credentials(
self,
config: CrossrefSourceConfig,
team_id: int,
schema_name: Optional[str] = None,
api_version: str | None = None,
) -> tuple[bool, str | None]:
if schema_name is not None and ENDPOINTS[schema_name].requires_scope and not self._has_scope(config):
return False, _SCOPE_REQUIRED_MESSAGE
if validate_crossref_credentials(config.mailto or None):
return True, None
return False, "Couldn't reach the Crossref API. Please try again."
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[CrossrefResumeConfig]:
return ResumableSourceManager[CrossrefResumeConfig](inputs, CrossrefResumeConfig)
def source_for_pipeline(
self,
config: CrossrefSourceConfig,
resumable_source_manager: ResumableSourceManager[CrossrefResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
if (
inputs.schema_name is not None
and ENDPOINTS[inputs.schema_name].requires_scope
and not self._has_scope(config)
):
raise ValueError(_SCOPE_REQUIRED_MESSAGE)
return crossref_source(
endpoint=inputs.schema_name,
logger=inputs.logger,
resumable_source_manager=resumable_source_manager,
mailto=config.mailto or None,
member_id=config.member_id or None,
funder_id=config.funder_id or None,
issn=config.issn or None,
should_use_incremental_field=inputs.should_use_incremental_field,
db_incremental_field_last_value=inputs.db_incremental_field_last_value
if inputs.should_use_incremental_field
else None,
incremental_field=inputs.incremental_field,
)