Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): sync reports with GitHub issue changes (#119915)

posthog[bot]dca366f52aa2
diff --git a/products/signals/backend/emission/github_issues.py b/products/signals/backend/emission/github_issues.py
index d2c92f90824e..e0f634bbbc73 100644
--- a/products/signals/backend/emission/github_issues.py
+++ b/products/signals/backend/emission/github_issues.py
@@ -1,4 +1,5 @@
import json
+import hashlib
from typing import Any
from structlog import get_logger
@@ -8,8 +9,6 @@ from products.signals.backend.emission.registry import SignalEmitterOutput, Sign
logger = get_logger(__name__)
-GITHUB_IGNORED_STATES = ("closed",)
-
GITHUB_SUMMARIZATION_PROMPT = """Summarize this GitHub issue for semantic search.
Output exactly two parts separated by a newline:
1. A short title (under 100 characters) that captures the core issue
@@ -86,21 +85,34 @@ def github_issue_emitter(team_id: int, record: dict[str, Any]) -> SignalEmitterO
msg = f"GitHub issue record has empty required field: id={issue_id!r}, title={title!r}"
logger.exception(msg, record=_loggable(record), team_id=team_id, signals_type="data-import-signals")
raise ValueError(msg)
- if not body:
- logger.info(
- "Ignoring GitHub issue without a body",
- record=_loggable(record),
- team_id=team_id,
- signals_type="data-import-signals",
- )
- return None
+ body = body or ""
+ extra = _build_extra(record)
+ labels = sorted(extra["labels"])
+ material_version = hashlib.sha256(
+ json.dumps(
+ {
+ "title": title,
+ "body": body,
+ "labels": labels,
+ "state": extra.get("state"),
+ "locked": extra.get("locked"),
+ },
+ sort_keys=True,
+ separators=(",", ":"),
+ ).encode()
+ ).hexdigest()
+ metadata = [f"State: {extra.get('state') or 'unknown'}"]
+ if labels:
+ metadata.append(f"Labels: {', '.join(labels)}")
return SignalEmitterOutput(
source_product="github",
source_type="issue",
source_id=str(issue_id),
- description=f"{title}\n{body}",
+ description=f"{title}\n{body}\n\n" + "\n".join(metadata),
weight=1.0,
- extra=_build_extra(record),
+ extra=extra,
+ idempotency_key=f"{issue_id}:{material_version}",
+ existing_source_only=not body or extra.get("state") == "closed",
)
@@ -159,10 +171,10 @@ GITHUB_ISSUES_CONFIG = SignalSourceTableConfig(
source_type="issue",
emitter=github_issue_emitter,
record_fetcher=data_warehouse_record_fetcher,
- partition_field="created_at",
+ sync_with_source=True,
+ partition_field="updated_at",
partition_field_is_datetime_string=True,
fields=REQUIRED_FIELDS + PASSTHROUGH_FIELDS + AUTHOR_FIELDS,
- where_clause=f"state NOT IN ({', '.join(repr(s) for s in GITHUB_IGNORED_STATES)})",
max_records=1000,
first_sync_lookback_days=1, # 24 hours
actionability_prompt=GITHUB_ACTIONABILITY_PROMPT,
diff --git a/products/signals/backend/emission/pipeline.py b/products/signals/backend/emission/pipeline.py
index 4d148c11edb0..a7a01068156f 100644
--- a/products/signals/backend/emission/pipeline.py
+++ b/products/signals/backend/emission/pipeline.py
@@ -38,6 +38,7 @@ from products.signals.backend.system_one_prompts import SystemOnePrompt, bundled
from products.signals.backend.temporal import metrics
from products.signals.backend.temporal.drop_telemetry import summarize_drop_error
from products.signals.backend.temporal.llm import effort_kwargs
+from products.signals.backend.temporal.signal_queries import fetch_source_signal_matches
logger = structlog.get_logger(__name__)
@@ -543,6 +544,7 @@ async def _emit_signals(
outputs: list[SignalEmitterOutput],
extra: dict[str, Any],
record_processed_outputs: bool = False,
+ sync_with_source: bool = False,
) -> int:
semaphore = asyncio.Semaphore(EMIT_CONCURRENCY_LIMIT)
_safe_heartbeat()
@@ -580,7 +582,16 @@ async def _emit_signals(
description=output.description,
weight=output.weight,
extra=output.extra,
- idempotency_key=output.source_id if record_processed_outputs else None,
+ idempotency_key=(
+ output.idempotency_key
+ if output.idempotency_key is not None
+ else output.source_id
+ if record_processed_outputs
+ else None
+ ),
+ sync_with_source=sync_with_source,
+ source_match_signal_id=output.source_match_signal_id,
+ source_match_report_id=output.source_match_report_id,
)
if record_processed_outputs:
await _record_processed_outputs(team, [output])
@@ -642,7 +653,7 @@ async def run_signal_pipeline(
unloggable_fields=config.unloggable_fields,
)
emission_run_id = activity.info().workflow_run_id if activity.in_activity() else None
- if config.record_processed_outputs or emission_run_id is not None:
+ if config.record_processed_outputs or emission_run_id is not None or any(o.idempotency_key for o in outputs):
outputs = [
dataclasses.replace(
output,
@@ -650,9 +661,13 @@ async def run_signal_pipeline(
team_id=team.id,
source_product=output.source_product,
source_type=output.source_type,
- idempotency_key=output.source_id
- if config.record_processed_outputs
- else f"{emission_run_id}:{activity.info().activity_id}:{output.source_id}",
+ idempotency_key=(
+ output.idempotency_key
+ if output.idempotency_key is not None
+ else output.source_id
+ if config.record_processed_outputs
+ else f"{emission_run_id}:{activity.info().activity_id}:{output.source_id}"
+ ),
),
)
for output in outputs
@@ -670,6 +685,36 @@ async def run_signal_pipeline(
for output in outputs:
capture_pipeline_stage("signal_data_source_entered", team, organization, output)
+ if config.sync_with_source and outputs:
+ source_matches = await database_sync_to_async(fetch_source_signal_matches, thread_sensitive=False)(
+ team,
+ [output.source_id for output in outputs],
+ config.source_product,
+ config.source_type,
+ )
+ outputs = [
+ dataclasses.replace(
+ output,
+ source_match_signal_id=source_matches[output.source_id][0].signal_id,
+ source_match_report_id=source_matches[output.source_id][0].report_id,
+ )
+ if output.source_id in source_matches
+ else output
+ for output in outputs
+ ]
+ for output in outputs:
+ if output.existing_source_only and output.source_match_report_id is None:
+ capture_pipeline_stage(
+ "signal_data_source_filtered",
+ team,
+ organization,
+ output,
+ {"filter_reason": "source_state_without_report"},
+ )
+ outputs = [
+ output for output in outputs if not output.existing_source_only or output.source_match_report_id is not None
+ ]
+
if config.summarization_prompt is not None and config.description_summarization_threshold_chars is not None:
threshold = config.description_summarization_threshold_chars
pre_summary_by_id = {o.source_id: o for o in outputs}
@@ -687,10 +732,10 @@ async def run_signal_pipeline(
if config.actionability_prompt:
steering = steering_from_config(source_config)
- pre_filter_by_id = {o.source_id: o for o in outputs}
- outputs = await filter_actionable(
+ pre_filter_by_id = {o.source_id: o for o in outputs if o.source_match_report_id is None}
+ filtered_outputs = await filter_actionable(
team=team,
- outputs=outputs,
+ outputs=list(pre_filter_by_id.values()),
actionability_prompt=config.actionability_prompt,
extra=extra,
# Steered teams get all of `extra`; everyone else gets only what the source declared, so
@@ -699,7 +744,12 @@ async def run_signal_pipeline(
context_fields=config.actionability_context_fields,
steering=steering,
)
- post_filter_ids = {o.source_id for o in outputs}
+ post_filter_ids = {o.source_id for o in filtered_outputs}
+ outputs = [
+ output
+ for output in outputs
+ if output.source_match_report_id is not None or output.source_id in post_filter_ids
+ ]
if config.record_processed_outputs:
await _record_processed_outputs(
team, [output for source_id, output in pre_filter_by_id.items() if source_id not in post_filter_ids]
@@ -720,6 +770,7 @@ async def run_signal_pipeline(
outputs=outputs,
extra=extra,
record_processed_outputs=config.record_processed_outputs,
+ sync_with_source=config.sync_with_source,
)
logger.info(f"Emitted {signals_emitted} signals for {source_label}", **extra)
return {"status": "success", "signals_emitted": signals_emitted}
diff --git a/products/signals/backend/emission/registry.py b/products/signals/backend/emission/registry.py
index bb2243e134db..5e4c35bb0c79 100644
--- a/products/signals/backend/emission/registry.py
+++ b/products/signals/backend/emission/registry.py
@@ -26,6 +26,10 @@ class SignalEmitterOutput:
weight: float
extra: dict[str, Any]
signal_id: str | None = None
+ idempotency_key: str | None = None
+ existing_source_only: bool = False
+ source_match_signal_id: str | None = None
+ source_match_report_id: str | None = None
# Type for signal emitter functions (None if the source has not enough meaningful data)
@@ -55,6 +59,9 @@ class SignalSourceTableConfig(BaseModel):
record_fetcher: RecordFetcher
# Snapshot sources deduplicate only after successful emission or an intentional filter.
record_processed_outputs: bool = False
+ # Re-emissions from the same source record must return to its existing report instead of relying
+ # on semantic matching. Use with an emitter-provided, material-version idempotency key.
+ sync_with_source: bool = False
# Field used to filter records by time window (e.g. "created_at")
partition_field: str
# Columns to SELECT — only what the emitter and extra metadata need
diff --git a/products/signals/backend/emission/tests/test_emit_signals.py b/products/signals/backend/emission/tests/test_emit_signals.py
index 6cf2f6b6ee0a..74db1f8c8ad1 100644
--- a/products/signals/backend/emission/tests/test_emit_signals.py
+++ b/products/signals/backend/emission/tests/test_emit_signals.py
@@ -54,6 +54,7 @@ def _make_config(**overrides: Any) -> SignalSourceTableConfig:
description=record.get("description", ""),
weight=0.5,
extra=record,
+ existing_source_only=record.get("existing_source_only", False),
),
"record_fetcher": data_warehouse_record_fetcher,
"partition_field": "created_at",
@@ -393,6 +394,36 @@ class TestEmissionRetryIdentity:
await run_signal_pipeline(team=team, config=config, records=[{"id": "record-1"}], extra={})
assert emit.call_args.kwargs["signal_id"] != first
+ @pytest.mark.asyncio
+ async def test_material_versions_dedupe_across_imports(self):
+ def emitter(team_id, record):
+ return replace(_make_output(source_id=record["id"]), idempotency_key=f"{record['id']}:{record['version']}")
+
+ config = _make_config(emitter=emitter, sync_with_source=True)
+ team = MagicMock(id=1)
+ with (
+ patch(f"{PIPELINE_MODULE_PATH}.activity") as temporal_activity,
+ patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as emit,
+ patch(f"{PIPELINE_MODULE_PATH}.fetch_source_signal_matches", return_value={}),
+ patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture"),
+ ):
+ temporal_activity.in_activity.return_value = True
+ temporal_activity.info.return_value.workflow_run_id = "first-import"
+ temporal_activity.info.return_value.activity_id = "emit-1"
+ await run_signal_pipeline(team=team, config=config, records=[{"id": "record-1", "version": "v1"}], extra={})
+ first = emit.call_args.kwargs
+ temporal_activity.info.return_value.workflow_run_id = "next-import"
+ await run_signal_pipeline(team=team, config=config, records=[{"id": "record-1", "version": "v1"}], extra={})
+ repeated = emit.call_args.kwargs
+ await run_signal_pipeline(team=team, config=config, records=[{"id": "record-1", "version": "v2"}], extra={})
+ changed = emit.call_args.kwargs
+
+ assert repeated["signal_id"] == first["signal_id"]
+ assert repeated["idempotency_key"] == first["idempotency_key"] == "record-1:v1"
+ assert changed["signal_id"] != first["signal_id"]
+ assert changed["idempotency_key"] == "record-1:v2"
+ assert changed["sync_with_source"] is True
+
class TestCheckActionability:
@pytest.mark.asyncio
@@ -723,8 +754,34 @@ class TestEmitSignals:
extra={},
idempotency_key=None,
signal_id=output.signal_id,
+ sync_with_source=False,
+ source_match_signal_id=None,
+ source_match_report_id=None,
)
+ @pytest.mark.asyncio
+ async def test_passes_source_sync_to_emit_signal(self):
+ output = replace(
+ _make_output(source_id="42", description="updated bug report"),
+ signal_id=str(uuid.uuid4()),
+ idempotency_key="42:material-version",
+ )
+
+ with (
+ patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as emit,
+ patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ ):
+ await _emit_signals(
+ team=MagicMock(),
+ organization=MagicMock(),
+ outputs=[output],
+ extra={},
+ sync_with_source=True,
+ )
+
+ assert emit.call_args.kwargs["idempotency_key"] == "42:material-version"
+ assert emit.call_args.kwargs["sync_with_source"] is True
+
@pytest.mark.asyncio
async def test_continues_on_individual_emit_failure(self):
outputs = [_make_output(source_id="1"), _make_output(source_id="2"), _make_output(source_id="3")]
@@ -799,6 +856,45 @@ class TestEmitSignals:
await _emit_signals(team=MagicMock(), organization=MagicMock(), outputs=outputs, extra={})
+class TestExistingSourceOnlyOutputs:
+ @pytest.mark.asyncio
+ async def test_terminal_source_only_emits_when_it_already_has_a_report(self):
+ config = _make_config(sync_with_source=True, actionability_prompt="Actionable? {description}")
+ records: list[dict[str, Any]] = [
+ {"id": "open", "state": "open"},
+ {"id": "known-closed", "state": "closed", "existing_source_only": True},
+ {"id": "new-closed", "state": "closed", "existing_source_only": True},
+ ]
+ team = MagicMock(id=1)
+ team.uuid = uuid.UUID("00000000-0000-0000-0000-000000000001")
+ with (
+ patch(
+ f"{PIPELINE_MODULE_PATH}.fetch_source_signal_matches",
+ return_value={"known-closed": [MagicMock(signal_id="previous-signal", report_id="existing-report")]},
+ ),
+ patch(f"{PIPELINE_MODULE_PATH}.emit_signal", new_callable=AsyncMock) as emit,
+ patch(
+ f"{PIPELINE_MODULE_PATH}.filter_actionable",
+ new_callable=AsyncMock,
+ side_effect=lambda **kwargs: kwargs["outputs"],
+ ) as filter_actionable,
+ patch(f"{PIPELINE_MODULE_PATH}.posthoganalytics.capture") as capture,
+ patch(f"{PIPELINE_MODULE_PATH}.activity"),
+ ):
+ result = await run_signal_pipeline(team=team, config=config, records=records, extra={})
+
+ assert result["signals_emitted"] == 2
+ assert {call.kwargs["source_id"] for call in emit.await_args_list} == {"open", "known-closed"}
+ known_call = next(call for call in emit.await_args_list if call.kwargs["source_id"] == "known-closed")
+ assert known_call.kwargs["source_match_signal_id"] == "previous-signal"
+ assert known_call.kwargs["source_match_report_id"] == "existing-report"
+ assert filter_actionable.await_args is not None
+ assert [output.source_id for output in filter_actionable.await_args.kwargs["outputs"]] == ["open"]
+ filtered = [call for call in capture.call_args_list if call.kwargs["event"] == "signal_data_source_filtered"]
+ assert [call.kwargs["properties"]["source_id"] for call in filtered] == ["new-closed"]
+ assert filtered[0].kwargs["properties"]["filter_reason"] == "source_state_without_report"
+
+
class TestPipelineStageTelemetry:
@pytest.mark.asyncio
async def test_captures_each_stage_per_signal(self):
diff --git a/products/signals/backend/emission/tests/test_github_issues.py b/products/signals/backend/emission/tests/test_github_issues.py
index 6ac3f0a5fdf4..eb61ff976cf6 100644
--- a/products/signals/backend/emission/tests/test_github_issues.py
+++ b/products/signals/backend/emission/tests/test_github_issues.py
@@ -2,10 +2,7 @@ import pytest
from unittest.mock import patch
from products.signals.backend.contracts import GithubIssueSignalExtra
-from products.signals.backend.emission import (
- github_issues as github_issues_module,
- pipeline,
-)
+from products.signals.backend.emission import pipeline
from products.signals.backend.emission.github_issues import EXTRA_FIELDS, GITHUB_ISSUES_CONFIG, github_issue_emitter
from products.signals.backend.emission.pipeline import build_emitter_outputs
@@ -21,6 +18,36 @@ class TestGithubIssueEmitter:
assert result.weight == 1.0
assert "Charts fail to render" in result.description
assert "special characters" in result.description
+ assert "State: open" in result.description
+ assert "Labels: bug, frontend" in result.description
+
+ def test_material_version_ignores_non_material_updates(self, github_issue_record):
+ first = github_issue_emitter(team_id=1, record=github_issue_record)
+ github_issue_record["updated_at"] = "2026-10-19T12:00:00Z"
+ repeated = github_issue_emitter(team_id=1, record=github_issue_record)
+
+ assert first is not None
+ assert repeated is not None
+ assert repeated.idempotency_key == first.idempotency_key
+
+ @pytest.mark.parametrize(
+ ("field", "value"),
+ [
+ ("title", "Charts fail differently"),
+ ("body", "A new reproduction"),
+ ("labels", '[{"name": "regression"}]'),
+ ("state", "closed"),
+ ("locked", True),
+ ],
+ )
+ def test_material_version_changes_with_report_content(self, github_issue_record, field, value):
+ first = github_issue_emitter(team_id=1, record=github_issue_record)
+ github_issue_record[field] = value
+ changed = github_issue_emitter(team_id=1, record=github_issue_record)
+
+ assert first is not None
+ assert changed is not None
+ assert changed.idempotency_key != first.idempotency_key
def test_includes_body_in_description(self, github_issue_record):
result = github_issue_emitter(team_id=1, record=github_issue_record)
@@ -29,11 +56,13 @@ class TestGithubIssueEmitter:
assert "blank white area" in result.description
@pytest.mark.parametrize("body", [None, ""])
- def test_skips_issue_with_no_body(self, github_issue_record, body):
+ def test_body_removal_emits_an_existing_source_only_revision(self, github_issue_record, body):
github_issue_record["body"] = body
result = github_issue_emitter(team_id=1, record=github_issue_record)
- assert result is None
+ assert result is not None
+ assert result.existing_source_only is True
+ assert "Charts fail to render" in result.description
@pytest.mark.parametrize("missing_field", ["id", "title"])
def test_raises_when_required_field_falsy(self, github_issue_record, missing_field):
@@ -107,15 +136,13 @@ class TestGithubIssueEmitter:
assert result.extra["author_login"] is None
assert result.extra["author_association"] is None
- def test_logged_records_leave_the_user_object_out(self, github_issue_record):
- # The record reaches the logs on every skipped or malformed issue, and the nested user object
- # carries a numeric id and avatar URLs that identify a person well beyond the handle.
+ def test_body_removal_leaves_the_user_object_out_of_extra(self, github_issue_record):
github_issue_record["body"] = ""
- with patch.object(github_issues_module.logger, "info") as log_info:
- github_issue_emitter(team_id=1, record=github_issue_record)
+ result = github_issue_emitter(team_id=1, record=github_issue_record)
- assert "user" not in log_info.call_args.kwargs["record"]
+ assert result is not None
+ assert "user" not in result.extra
def test_a_failed_record_reaches_the_pipeline_log_without_the_user_object(self, github_issue_record):
# The emitter raises on malformed labels, and the shared pipeline logs the record it handed
@@ -165,12 +192,21 @@ class TestGithubIssueEmitter:
class TestGithubIssuesConfig:
def test_partition_field(self):
- assert GITHUB_ISSUES_CONFIG.partition_field == "created_at"
-
- def test_where_clause_excludes_closed(self):
- assert GITHUB_ISSUES_CONFIG.where_clause is not None
- assert "closed" in GITHUB_ISSUES_CONFIG.where_clause
- assert "NOT IN" in GITHUB_ISSUES_CONFIG.where_clause
+ assert GITHUB_ISSUES_CONFIG.partition_field == "updated_at"
+
+ def test_closed_issues_only_refresh_existing_reports(self):
+ assert GITHUB_ISSUES_CONFIG.sync_with_source is True
+ assert GITHUB_ISSUES_CONFIG.where_clause is None
+
+ record = {
+ "id": 1,
+ "title": "Closed issue",
+ "body": "Resolved",
+ "state": "closed",
+ }
+ result = github_issue_emitter(team_id=1, record=record)
+ assert result is not None
+ assert result.existing_source_only is True
def test_has_actionability_prompt(self):
assert GITHUB_ISSUES_CONFIG.actionability_prompt is not None
diff --git a/products/signals/backend/emission/tests/test_google_search_console_opportunities.py b/products/signals/backend/emission/tests/test_google_search_console_opportunities.py
index d52655faf883..1fe50289fd70 100644
--- a/products/signals/backend/emission/tests/test_google_search_console_opportunities.py
+++ b/products/signals/backend/emission/tests/test_google_search_console_opportunities.py
@@ -126,8 +126,7 @@ class TestGoogleSearchConsoleEmitter:
assert variant is not None
# extra="forbid" + strict types on the contract catches any drift between emitter and schema.
data = dataclasses.asdict(output)
- data.pop("signal_id")
- variant.model_validate(data)
+ variant.model_validate({field: data[field] for field in variant.model_fields if field in data})
class TestGoogleSearchConsoleConfig:
diff --git a/products/signals/backend/emission/tests/test_schema_validation.py b/products/signals/backend/emission/tests/test_schema_validation.py
index 715f1d634bd3..203758be5b0c 100644
--- a/products/signals/backend/emission/tests/test_schema_validation.py
+++ b/products/signals/backend/emission/tests/test_schema_validation.py
@@ -22,11 +22,10 @@ def _load_fixture(filename: str) -> list[dict]:
def _validate_output(output):
data = dataclasses.asdict(output)
- data.pop("signal_id")
variant_type = SIGNAL_VARIANT_LOOKUP.get((output.source_product, output.source_type))
if variant_type is None:
raise ValueError(f"No SignalInput variant for ({output.source_product}, {output.source_type})")
- variant_type.model_validate(data)
+ variant_type.model_validate({field: data[field] for field in variant_type.model_fields if field in data})
GITHUB_RECORDS = _load_fixture("github_issues.json")
diff --git a/products/signals/backend/facade/api.py b/products/signals/backend/facade/api.py
index 3534c4189ff8..35e21de0efea 100644
--- a/products/signals/backend/facade/api.py
+++ b/products/signals/backend/facade/api.py
@@ -763,6 +763,9 @@ async def emit_signal(
remediation: SignalRemediation | None = None,
idempotency_key: str | None = None,
signal_id: str | None = None,
+ sync_with_source: bool = False,
+ source_match_signal_id: str | None = None,
+ source_match_report_id: str | None = None,
) -> None:
"""
Emit a signal for grouping and potential report generation, fire-and-forget.
@@ -795,6 +798,10 @@ async def emit_signal(
investigating from scratch. Not required by any existing source.
idempotency_key: Optional stable key for callers that may retry. Repeated calls with
the same key, source product, and source type start at most one emitter workflow.
+ sync_with_source: Route a new material version of the same source record back to its
+ existing report instead of relying on semantic matching.
+ source_match_signal_id: Existing signal matched by durable source identity, when known.
+ source_match_report_id: Existing report matched by durable source identity, when known.
Example:
await emit_signal(
@@ -913,6 +920,9 @@ async def emit_signal(
weight=weight,
extra=extra or {},
remediation=remediation_dict,
+ sync_with_source=sync_with_source,
+ source_match_signal_id=source_match_signal_id,
+ source_match_report_id=source_match_report_id,
)
# Ensure the buffer workflow is running (idempotent)
diff --git a/products/signals/backend/migrations/0195_signalemissionrecord_source_assignment.py b/products/signals/backend/migrations/0195_signalemissionrecord_source_assignment.py
new file mode 100644
index 000000000000..9b6e3e3068ae
--- /dev/null
+++ b/products/signals/backend/migrations/0195_signalemissionrecord_source_assignment.py
@@ -0,0 +1,22 @@
+# Generated by Django 5.2.18 on 2026-10-19 22:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("signals", "0194_report_monitoring_attempt"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="signalemissionrecord",
+ name="report_id",
+ field=models.CharField(blank=True, max_length=200, null=True),
+ ),
+ migrations.AddField(
+ model_name="signalemissionrecord",
+ name="signal_id",
+ field=models.CharField(blank=True, max_length=200, null=True),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 5631ca866075..50a4464b8108 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0194_report_monitoring_attempt
+0195_signalemissionrecord_source_assignment
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 74cdf9196213..dab01b405217 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -1334,6 +1334,8 @@ class SignalEmissionRecord(UUIDModel):
source_type = models.CharField(max_length=100)
source_id = models.CharField(max_length=200)
emitted_at = models.DateTimeField()
+ signal_id = models.CharField(max_length=200, null=True, blank=True)
+ report_id = models.CharField(max_length=200, null=True, blank=True)
class Meta:
constraints = [
diff --git a/products/signals/backend/temporal/grouping.py b/products/signals/backend/temporal/grouping.py
index c4781f4ad6d8..61b5c53af9e4 100644
--- a/products/signals/backend/temporal/grouping.py
+++ b/products/signals/backend/temporal/grouping.py
@@ -35,7 +35,7 @@ from products.signals.backend.artefact_schemas import MAX_REPORT_LINK_REASON_LEN
from products.signals.backend.billing import BILLING_EXEMPT_SOURCE_PRODUCTS
from products.signals.backend.daily_limit import capture_signal_report_daily_limit_paused, daily_report_limit_gate
from products.signals.backend.enums import ReportLinkKind, ReportLinkWritePath, SignalSourceProduct
-from products.signals.backend.models import SignalReport, SignalReportArtefact
+from products.signals.backend.models import SignalEmissionRecord, SignalReport, SignalReportArtefact
from products.signals.backend.quota import capture_signal_report_quota_paused, self_driving_quota_gate
from products.signals.backend.receivers import _is_safety_suppressed
from products.signals.backend.recurrence import fixed_dismissal_at
@@ -61,6 +61,7 @@ from products.signals.backend.temporal.signal_queries import (
WaitForClickHouseSignal,
fetch_signal_type_examples_activity,
fetch_signals_for_report_activity,
+ fetch_source_signal_matches,
run_signal_semantic_search_activity,
soft_delete_report_signals,
wait_for_signal_in_clickhouse_activity,
@@ -462,6 +463,10 @@ class MatchSignalToReportInput:
# Optional with a default for deploy-time backward compatibility (see GenerateSearchQueriesInput).
team_id: int | None = None
signal_id: str | None = None
+ source_id: str | None = None
+ sync_with_source: bool = False
+ source_match_signal_id: str | None = None
+ source_match_report_id: str | None = None
async def match_signal_to_report(input: MatchSignalToReportInput) -> MatchResult:
@@ -532,7 +537,38 @@ async def match_signal_to_report(input: MatchSignalToReportInput) -> MatchResult
async def match_signal_to_report_activity(input: MatchSignalToReportInput) -> MatchResult:
"""Determine if a new signal matches an existing report or needs a new one."""
try:
- result = await match_signal_to_report(input)
+ result: MatchResult
+ if input.source_match_signal_id and input.source_match_report_id:
+ result = ExistingReportMatch(
+ report_id=input.source_match_report_id,
+ match_metadata=MatchedMetadata(
+ parent_signal_id=input.source_match_signal_id,
+ match_query="source identity",
+ reason="Material update to the same source record",
+ ),
+ source_synced=True,
+ )
+ elif input.sync_with_source and input.team_id is not None and input.source_id:
+ team = await Team.objects.aget(pk=input.team_id)
+ source_matches = await database_sync_to_async(fetch_source_signal_matches, thread_sensitive=False)(
+ team, [input.source_id], input.source_product, input.source_type
+ )
+ matches = source_matches.get(input.source_id, [])
+ if matches:
+ source_match = matches[0]
+ result = ExistingReportMatch(
+ report_id=source_match.report_id,
+ match_metadata=MatchedMetadata(
+ parent_signal_id=source_match.signal_id,
+ match_query="source identity",
+ reason="Material update to the same source record",
+ ),
+ source_synced=True,
+ )
+ else:
+ result = await match_signal_to_report(input)
+ else:
+ result = await match_signal_to_report(input)
if isinstance(result, ExistingReportMatch) and input.team_id is not None:
report = await SignalReport.objects.filter(team_id=input.team_id, id=result.report_id).afirst()
if report is not None and report.status != SignalReport.Status.DELETED:
@@ -555,6 +591,8 @@ async def match_signal_to_report_activity(input: MatchSignalToReportInput) -> Ma
f"Failed to match signal with LLM: {e}",
source_product=input.source_product,
source_type=input.source_type,
+ source_id=input.source_id,
+ sync_with_source=input.sync_with_source,
)
raise
@@ -727,7 +765,7 @@ async def verify_match_specificity_activity(input: VerifyMatchSpecificityInput)
raise
-@dataclass
+@dataclass(frozen=False)
class AssignAndEmitSignalInput:
team_id: int
signal_id: str
@@ -742,6 +780,7 @@ class AssignAndEmitSignalInput:
timestamp: Optional[datetime] = None
updated_title: Optional[str] = None
remediation: Optional[dict] = None
+ sync_with_source: bool = False
@dataclass(frozen=False)
@@ -934,8 +973,8 @@ async def assign_and_emit_signal_activity(input: AssignAndEmitSignalInput) -> As
# - POTENTIAL: promote once total_weight >= WEIGHT_THRESHOLD and signal_count >= signals_at_run
# (snooze gate, defaults to 0). Uncapped — a report's first research always runs.
# - READY: re-research once the report has reached its next bucket in
- # RESEARCH_SIGNAL_BUCKETS. Between buckets, and past the last one, signals are
- # collected, not researched.
+ # RESEARCH_SIGNAL_BUCKETS. Material revisions from a report's source bypass the
+ # buckets so the researched content stays current.
# - CANDIDATE: re-promote to self-heal failed spawns (uncapped; concurrent runs blocked by Temporal).
#
# The bucket is measured against what the last completed pass actually covered, so a
@@ -946,7 +985,10 @@ async def assign_and_emit_signal_activity(input: AssignAndEmitSignalInput) -> As
signals_researched = report.researched_signal_count
bucket = next_research_bucket(signals_researched)
is_reresearch = report.status == SignalReport.Status.READY
- reresearch_capped = is_reresearch and (bucket is None or report.signal_count < bucket)
+ source_refresh = isinstance(match_result, ExistingReportMatch) and match_result.source_synced
+ reresearch_capped = (
+ is_reresearch and not source_refresh and (bucket is None or report.signal_count < bucket)
+ )
if (
(is_reresearch and not reresearch_capped)
or report.status == SignalReport.Status.CANDIDATE
@@ -977,13 +1019,24 @@ async def assign_and_emit_signal_activity(input: AssignAndEmitSignalInput) -> As
if (
report.status == SignalReport.Status.IN_PROGRESS
and report.pending_triggering_signal_id is None
- and pending_bucket is not None
- and report.signal_count >= pending_bucket
+ and (source_refresh or (pending_bucket is not None and report.signal_count >= pending_bucket))
):
report.pending_triggering_signal_id = input.signal_id
report.save(update_fields=["pending_triggering_signal_id"])
report_id = str(report.id)
+ if input.sync_with_source:
+ SignalEmissionRecord.objects.update_or_create(
+ team_id=input.team_id,
+ source_product=input.source_product,
+ source_type=input.source_type,
+ source_id=input.source_id,
+ defaults={
+ "emitted_at": timezone.now(),
+ "signal_id": input.signal_id,
+ "report_id": report_id,
+ },
+ )
_link_check_follow_up(
team_id=input.team_id,
report_id=report_id,
@@ -1095,6 +1148,21 @@ async def assign_and_emit_signal_activity(input: AssignAndEmitSignalInput) -> As
},
groups=groups(team.organization, team),
)
+ if isinstance(match_result, ExistingReportMatch) and match_result.source_synced:
+ posthoganalytics.capture(
+ event="signal_source_report_refresh_requested",
+ distinct_id=str(team.uuid),
+ properties={
+ "source_product": input.source_product,
+ "source_type": input.source_type,
+ "source_id": input.source_id,
+ "report_id": db_result.report_id,
+ "previous_report_id": match_result.report_id,
+ "created_recurrence": db_result.report_id != match_result.report_id,
+ "promoted": db_result.promoted,
+ },
+ groups=groups(team.organization, team),
+ )
except Exception as e:
posthoganalytics.capture_exception(e)
# Swallow the exception, to avoid breaking the flow over failed analytics event
@@ -1271,27 +1339,33 @@ async def _process_signal_batch(
# === PARALLEL PHASE (steps 1-4) ===
try:
+ semantic_indices = [
+ i for i, signal in enumerate(batch) if not (signal.source_match_signal_id and signal.source_match_report_id)
+ ]
# Step 1a: Fetch type examples if not cached (needed by query gen)
if cached_type_examples is not None:
type_examples_result = cached_type_examples
- else:
+ elif semantic_indices:
type_examples_result = await workflow.execute_activity(
fetch_signal_type_examples_activity,
FetchSignalTypeExamplesInput(team_id=team_id),
start_to_close_timeout=timedelta(minutes=5),
retry_policy=RetryPolicy(maximum_attempts=3),
)
+ else:
+ type_examples_result = FetchSignalTypeExamplesOutput(examples=[])
# Step 1b: Embed all signals + generate search queries in parallel
# (query gen needs type examples but NOT the signal embeddings)
- query_inputs = batch
- query_result_indices = list(range(len(batch)))
+ semantic_signals = [batch[i] for i in semantic_indices]
+ query_inputs = semantic_signals
+ query_result_indices = list(range(len(semantic_signals)))
if isolate_refusals:
query_inputs = []
query_result_indices = []
payload_indices: dict[str, int] = {}
# Duplicate payloads must share a query activity so their refusal counts cannot race.
- for signal in batch:
+ for signal in semantic_signals:
payload = json.dumps(asdict(signal), sort_keys=True)
if payload not in payload_indices:
payload_indices[payload] = len(query_inputs)
@@ -1328,7 +1402,9 @@ async def _process_signal_batch(
)
signal_embeddings = cast(list[GenerateEmbeddingOutput], step1b_results[: len(batch)])
query_outputs = cast(list[GenerateSearchQueriesOutput], step1b_results[len(batch) :])
- query_gen_results = [query_outputs[i] for i in query_result_indices]
+ query_gen_results = [GenerateSearchQueriesOutput(queries=[]) for _ in batch]
+ for batch_index, query_index in zip(semantic_indices, query_result_indices, strict=True):
+ query_gen_results[batch_index] = query_outputs[query_index]
if isolate_refusals:
retained_indices = [i for i, result in enumerate(query_gen_results) if not result.quarantined]
@@ -1394,7 +1470,10 @@ async def _process_signal_batch(
per_signal_ch_results[sig_idx].append(all_search_results[flat_idx].candidates)
# Step 4.5: Fetch report contexts for all CH candidates (group-aware matching)
- all_candidate_report_ids = list({c.report_id for results in all_search_results for c in results.candidates})
+ all_candidate_report_ids = list(
+ {c.report_id for results in all_search_results for c in results.candidates}
+ | {signal.source_match_report_id for signal in batch if signal.source_match_report_id}
+ )
report_contexts_result: FetchReportContextsOutput = await workflow.execute_activity(
fetch_report_contexts_activity,
FetchReportContextsInput(team_id=team_id, report_ids=all_candidate_report_ids),
@@ -1460,6 +1539,10 @@ async def _process_signal_batch(
queries=per_signal_queries[i],
query_results=augmented_results,
report_contexts=report_contexts,
+ source_id=signal.source_id,
+ sync_with_source=signal.sync_with_source,
+ source_match_signal_id=signal.source_match_signal_id,
+ source_match_report_id=signal.source_match_report_id,
),
start_to_close_timeout=timedelta(minutes=10),
retry_policy=RetryPolicy(maximum_attempts=5),
@@ -1468,7 +1551,7 @@ async def _process_signal_batch(
# Step 5.5: PR-specificity verification for existing matches
updated_title: Optional[str] = None
- if isinstance(match_result, ExistingReportMatch):
+ if isinstance(match_result, ExistingReportMatch) and not match_result.source_synced:
report_ctx = report_contexts.get(match_result.report_id)
report_title = (
match_result.report_title
@@ -1536,6 +1619,7 @@ async def _process_signal_batch(
match_result=match_result,
updated_title=updated_title,
remediation=signal.remediation,
+ sync_with_source=signal.sync_with_source,
),
start_to_close_timeout=timedelta(minutes=5),
retry_policy=RetryPolicy(maximum_attempts=3),
diff --git a/products/signals/backend/temporal/parallel_grouping.py b/products/signals/backend/temporal/parallel_grouping.py
index 2bf260e879f3..25945cc36a4c 100644
--- a/products/signals/backend/temporal/parallel_grouping.py
+++ b/products/signals/backend/temporal/parallel_grouping.py
@@ -176,6 +176,10 @@ async def _process_signal(
queries=queries,
query_results=augmented_results,
report_contexts=report_contexts,
+ source_id=signal.source_id,
+ sync_with_source=signal.sync_with_source,
+ source_match_signal_id=signal.source_match_signal_id,
+ source_match_report_id=signal.source_match_report_id,
),
start_to_close_timeout=timedelta(minutes=10),
retry_policy=RetryPolicy(maximum_attempts=5),
@@ -184,7 +188,7 @@ async def _process_signal(
# Step 5.5: PR-specificity verification for existing matches
updated_title: Optional[str] = None
- if isinstance(match_result, ExistingReportMatch):
+ if isinstance(match_result, ExistingReportMatch) and not match_result.source_synced:
report_ctx = report_contexts.get(match_result.report_id)
report_title = report_ctx.title if report_ctx else ""
@@ -246,6 +250,7 @@ async def _process_signal(
match_result=match_result,
updated_title=updated_title,
remediation=signal.remediation,
+ sync_with_source=signal.sync_with_source,
),
start_to_close_timeout=timedelta(minutes=5),
retry_policy=RetryPolicy(maximum_attempts=3),
diff --git a/products/signals/backend/temporal/signal_queries.py b/products/signals/backend/temporal/signal_queries.py
index 64791f218d84..1e887e5883ba 100644
--- a/products/signals/backend/temporal/signal_queries.py
+++ b/products/signals/backend/temporal/signal_queries.py
@@ -16,11 +16,13 @@ from posthog.api.embedding_worker import DocumentKey, async_get_recently_seen_do
from posthog.clickhouse.client.connection import ClickHouseUser
from posthog.clickhouse.query_tagging import Feature, Product, tag_queries
from posthog.clickhouse.workload import Workload
+from posthog.dataclasses import frozen
from posthog.models import Team
from posthog.temporal.common.client import async_connect
from posthog.temporal.common.scoped import scoped_temporal
from posthog.temporal.common.utils import close_db_connections
+from products.signals.backend.models import SignalEmissionRecord, SignalReport
from products.signals.backend.signal_metadata import (
EMBEDDING_MODEL,
REASSIGN_SIGNAL_ROW_CAP,
@@ -852,9 +854,18 @@ def fetch_report_ids_for_source_ids(team: Team, source_ids: list[str]) -> dict[s
return {row[0]: row[1] for row in (result.results or []) if row[0] and row[1]}
-def fetch_live_report_ids_for_source_ids(
- team: Team, source_ids: list[str], source_product: str | None = None
-) -> dict[str, list[str]]:
+@frozen
+class SourceSignalMatch:
+ signal_id: str
+ report_id: str
+
+
+def fetch_live_source_signal_matches(
+ team: Team,
+ source_ids: list[str],
+ source_product: str | None = None,
+ source_type: str | None = None,
+) -> dict[str, list[SourceSignalMatch]]:
"""Map each `source_id` to every live report its signals grouped into, newest report first.
Sibling of `fetch_report_ids_for_source_ids`, which answers a different question: that one
@@ -872,23 +883,27 @@ def fetch_live_report_ids_for_source_ids(
source_id_scan_filter = "JSONExtractString(metadata, 'source_id') IN ({source_ids})"
product_filter = "AND source_product = {source_product}" if source_product is not None else ""
+ type_filter = "AND source_type = {source_type}" if source_type is not None else ""
ch_query = f"""
- SELECT source_id, groupArray(report_id) as report_ids
+ SELECT source_id, groupArray((signal_id, report_id)) as signal_matches
FROM (
SELECT
+ document_id as signal_id,
JSONExtractString(metadata, 'source_id') as source_id,
JSONExtractString(metadata, 'report_id') as report_id,
JSONExtractBool(metadata, 'deleted') as is_deleted,
JSONExtractString(metadata, 'source_product') as source_product,
+ JSONExtractString(metadata, 'source_type') as source_type,
max(timestamp) as latest_timestamp
FROM ({_deduped_signals_subquery(extra_where=source_id_scan_filter)})
- GROUP BY source_id, report_id, is_deleted, source_product
+ GROUP BY signal_id, source_id, report_id, is_deleted, source_product, source_type
ORDER BY latest_timestamp DESC
)
WHERE NOT is_deleted
AND source_id != ''
AND report_id != ''
{product_filter}
+ {type_filter}
GROUP BY source_id
"""
@@ -898,6 +913,8 @@ def fetch_live_report_ids_for_source_ids(
}
if source_product is not None:
placeholders["source_product"] = ast.Constant(value=source_product)
+ if source_type is not None:
+ placeholders["source_type"] = ast.Constant(value=source_type)
tag_queries(product=Product.SIGNALS, feature=Feature.QUERY)
result = execute_hogql_query(
@@ -907,4 +924,77 @@ def fetch_live_report_ids_for_source_ids(
placeholders=placeholders,
)
- return {row[0]: list(row[1]) for row in (result.results or []) if row[0] and row[1]}
+ return {
+ row[0]: [SourceSignalMatch(signal_id=signal_id, report_id=report_id) for signal_id, report_id in row[1]]
+ for row in (result.results or [])
+ if row[0] and row[1]
+ }
+
+
+def fetch_source_signal_matches(
+ team: Team,
+ source_ids: list[str],
+ source_product: str,
+ source_type: str,
+) -> dict[str, list[SourceSignalMatch]]:
+ """Resolve source identity durably, falling back to retained signal embeddings during rollout."""
+ if not source_ids:
+ return {}
+
+ records = list(
+ SignalEmissionRecord.objects.filter(
+ team=team,
+ source_product=source_product,
+ source_type=source_type,
+ source_id__in=source_ids,
+ signal_id__isnull=False,
+ report_id__isnull=False,
+ ).values_list("source_id", "signal_id", "report_id")
+ )
+ report_ids = {report_id for _, _, report_id in records if report_id}
+ live_report_ids = {
+ str(report_id)
+ for report_id in SignalReport.objects.filter(team=team, id__in=report_ids)
+ .exclude(status=SignalReport.Status.DELETED)
+ .values_list("id", flat=True)
+ }
+ matches = {
+ source_id: [SourceSignalMatch(signal_id=signal_id, report_id=report_id)]
+ for source_id, signal_id, report_id in records
+ if signal_id and report_id in live_report_ids
+ }
+
+ missing_source_ids = [source_id for source_id in source_ids if source_id not in matches]
+ if not missing_source_ids:
+ return matches
+
+ retained_matches = fetch_live_source_signal_matches(
+ team,
+ missing_source_ids,
+ source_product,
+ source_type,
+ )
+ retained_report_ids = {match.report_id for source_matches in retained_matches.values() for match in source_matches}
+ live_retained_report_ids = {
+ str(report_id)
+ for report_id in SignalReport.objects.filter(team=team, id__in=retained_report_ids)
+ .exclude(status=SignalReport.Status.DELETED)
+ .values_list("id", flat=True)
+ }
+ matches.update(
+ {
+ source_id: [match for match in source_matches if match.report_id in live_retained_report_ids]
+ for source_id, source_matches in retained_matches.items()
+ if any(match.report_id in live_retained_report_ids for match in source_matches)
+ }
+ )
+ return matches
+
+
+def fetch_live_report_ids_for_source_ids(
+ team: Team, source_ids: list[str], source_product: str | None = None
+) -> dict[str, list[str]]:
+ return {
+ source_id: [match.report_id for match in matches]
+ for source_id, matches in fetch_live_source_signal_matches(team, source_ids, source_product).items()
+ }
diff --git a/products/signals/backend/temporal/types.py b/products/signals/backend/temporal/types.py
index 6231c7c9c9f9..49d3fece50d3 100644
--- a/products/signals/backend/temporal/types.py
+++ b/products/signals/backend/temporal/types.py
@@ -75,6 +75,9 @@ class EmitSignalInputs:
# present; not required by any source.
remediation: Optional[dict] = None
signal_id: str | None = None
+ sync_with_source: bool = False
+ source_match_signal_id: str | None = None
+ source_match_report_id: str | None = None
@dataclass
@@ -132,6 +135,7 @@ class ExistingReportMatch:
report_id: str
match_metadata: MatchedMetadata
report_title: str | None = None
+ source_synced: bool = False
@dataclass
diff --git a/products/signals/backend/test/test_assign_and_emit_signal.py b/products/signals/backend/test/test_assign_and_emit_signal.py
index e7389c847f6a..9baf4011e0a3 100644
--- a/products/signals/backend/test/test_assign_and_emit_signal.py
+++ b/products/signals/backend/test/test_assign_and_emit_signal.py
@@ -17,7 +17,7 @@ from products.signals.backend.artefact_attribution import ArtefactAttribution
from products.signals.backend.artefact_schemas import Dismissal, RelatedTo, ReportLink
from products.signals.backend.daily_limit import DailyReportLimitGate
from products.signals.backend.enums import ReportLinkKind
-from products.signals.backend.models import SignalReport, SignalReportArtefact
+from products.signals.backend.models import SignalEmissionRecord, SignalReport, SignalReportArtefact
from products.signals.backend.quota import SelfDrivingQuotaGate
from products.signals.backend.report_merge import MERGE_DISMISSAL_REASON
from products.signals.backend.spend import record_llm_spend
@@ -31,6 +31,7 @@ from products.signals.backend.temporal.grouping import (
match_signal_to_report_activity,
verify_match_specificity_activity,
)
+from products.signals.backend.temporal.signal_queries import SourceSignalMatch
from products.signals.backend.temporal.types import (
ExistingReportMatch,
MatchedMetadata,
@@ -79,7 +80,7 @@ def patch_side_effects():
}
-def _existing_match(report_id: str) -> ExistingReportMatch:
+def _existing_match(report_id: str, *, source_synced: bool = False) -> ExistingReportMatch:
return ExistingReportMatch(
report_id=report_id,
match_metadata=MatchedMetadata(
@@ -87,6 +88,7 @@ def _existing_match(report_id: str) -> ExistingReportMatch:
match_query="test query",
reason="similar content",
),
+ source_synced=source_synced,
)
@@ -769,6 +771,76 @@ async def test_matching_uses_current_recurrence_before_specificity(ateam, delete
assert intermediate.signal_count == 0
+async def test_source_sync_matches_without_semantic_rejudgment(ateam):
+ report = await database_sync_to_async(SignalReport.objects.create)(
+ team=ateam, status=SignalReport.Status.READY, title="Existing source report"
+ )
+ source_match = SourceSignalMatch(signal_id=str(uuid.uuid4()), report_id=str(report.id))
+ with (
+ patch(f"{GROUPING_MODULE_PATH}.fetch_source_signal_matches", return_value={"source-1": [source_match]}),
+ patch(f"{GROUPING_MODULE_PATH}.match_signal_to_report", new=AsyncMock()) as semantic_match,
+ ):
+ match = await match_signal_to_report_activity(
+ MatchSignalToReportInput(
+ team_id=ateam.id,
+ description="Updated source content",
+ source_product="github",
+ source_type="issue",
+ source_id="source-1",
+ sync_with_source=True,
+ queries=[],
+ query_results=[],
+ report_contexts={},
+ )
+ )
+
+ assert isinstance(match, ExistingReportMatch)
+ assert match.report_id == str(report.id)
+ assert match.match_metadata.parent_signal_id == source_match.signal_id
+ assert match.source_synced is True
+ semantic_match.assert_not_awaited()
+
+
+async def test_source_refresh_bypasses_research_buckets_and_emits_telemetry(ateam, patch_side_effects):
+ report = await database_sync_to_async(SignalReport.objects.create)(
+ team=ateam,
+ status=SignalReport.Status.READY,
+ total_weight=2,
+ signal_count=2,
+ signals_researched=2,
+ )
+ input_ = _build_input(ateam.id, _existing_match(str(report.id), source_synced=True), weight=0.5)
+ input_.sync_with_source = True
+
+ result = await assign_and_emit_signal_activity(input_)
+
+ assert result.report_id == str(report.id)
+ assert result.promoted is True
+ await database_sync_to_async(report.refresh_from_db)()
+ assert report.status == SignalReport.Status.CANDIDATE
+ events = [call.kwargs["event"] for call in patch_side_effects["capture"].call_args_list]
+ assert "signal_source_report_refresh_requested" in events
+ refresh = next(
+ call.kwargs["properties"]
+ for call in patch_side_effects["capture"].call_args_list
+ if call.kwargs["event"] == "signal_source_report_refresh_requested"
+ )
+ assert refresh["report_id"] == str(report.id)
+ assert refresh["created_recurrence"] is False
+ association = await SignalEmissionRecord.objects.aget(
+ team_id=ateam.id,
+ source_product=input_.source_product,
+ source_type=input_.source_type,
+ source_id=input_.source_id,
+ )
+ assert association.signal_id == input_.signal_id
+ assert association.report_id == str(report.id)
+
+
@pytest.mark.parametrize(
("signals_researched", "expected"),
[
diff --git a/products/signals/backend/test/test_grouping_refusal_batch.py b/products/signals/backend/test/test_grouping_refusal_batch.py
index 14b1613d9af1..a0d6abf8055c 100644
--- a/products/signals/backend/test/test_grouping_refusal_batch.py
+++ b/products/signals/backend/test/test_grouping_refusal_batch.py
@@ -8,7 +8,13 @@ from products.signals.backend.temporal.signal_queries import (
FetchSignalTypeExamplesOutput,
RunSignalSemanticSearchOutput,
)
-from products.signals.backend.temporal.types import EmitSignalInputs, NewReportMatch, NoMatchMetadata
+from products.signals.backend.temporal.types import (
+ EmitSignalInputs,
+ ExistingReportMatch,
+ MatchedMetadata,
+ NewReportMatch,
+ NoMatchMetadata,
+)
@pytest.mark.asyncio
@@ -31,6 +37,18 @@ async def test_quarantined_signals_do_not_block_healthy_batch_members(
]
if duplicate:
batch.append(batch[0])
+ batch.append(
+ EmitSignalInputs(
+ team_id=1,
+ source_product="github",
+ source_type="issue",
+ source_id="source-sync",
+ description="2",
+ sync_with_source=True,
+ source_match_signal_id="previous-signal",
+ source_match_report_id="existing-report",
+ )
+ )
query_requests: list[str] = []
emitted: list[grouping.AssignAndEmitSignalInput] = []
@@ -52,6 +70,19 @@ async def test_quarantined_signals_do_not_block_healthy_batch_members(
return grouping.FetchReportContextsOutput(contexts={})
if activity == grouping.match_signal_to_report_activity:
assert isinstance(input, grouping.MatchSignalToReportInput)
+ if input.source_id == "source-sync":
+ assert input.queries == []
+ assert input.source_match_signal_id == "previous-signal"
+ assert input.source_match_report_id == "existing-report"
+ return ExistingReportMatch(
+ report_id="existing-report",
+ match_metadata=MatchedMetadata(
+ parent_signal_id="previous-signal",
+ match_query="source identity",
+ reason="source update",
+ ),
+ source_synced=True,
+ )
assert input.queries == [input.description]
return NewReportMatch(
title="Example report", summary="Example summary", match_metadata=NoMatchMetadata(reason="new")
@@ -77,8 +108,10 @@ async def test_quarantined_signals_do_not_block_healthy_batch_members(
):
dropped, _ = await grouping._process_signal_batch(batch, FetchSignalTypeExamplesOutput(examples=[]))
- assert dropped == sum(quarantined[int(signal.source_id)] for signal in batch)
+ assert dropped == sum(quarantined[int(signal.source_id)] for signal in batch if signal.source_id != "source-sync")
assert [signal.source_id for signal in emitted] == [
- signal.source_id for signal in batch if not quarantined[int(signal.source_id)]
+ signal.source_id
+ for signal in batch
+ if signal.source_id == "source-sync" or not quarantined[int(signal.source_id)]
]
assert len(query_requests) == 2
diff --git a/products/signals/backend/test/test_signal_queries.py b/products/signals/backend/test/test_signal_queries.py
index ee560f4383f6..d803c70c6979 100644
--- a/products/signals/backend/test/test_signal_queries.py
+++ b/products/signals/backend/test/test_signal_queries.py
@@ -24,7 +24,12 @@ from products.signals.backend.facade.api import (
get_outcomes_for_signal_source_slice,
get_reports_for_signal_source_slice,
)
-from products.signals.backend.models import SignalReport, SignalReportArtefact, SignalReportPullRequest
+from products.signals.backend.models import (
+ SignalEmissionRecord,
+ SignalReport,
+ SignalReportArtefact,
+ SignalReportPullRequest,
+)
from products.signals.backend.signal_metadata import (
EMBEDDING_MODEL,
ReportSignalMeta,
@@ -38,8 +43,11 @@ from products.signals.backend.signal_metadata import (
from products.signals.backend.spend import SignalMetadataChange, SignalSpendUpdate, record_llm_spend
from products.signals.backend.spend_metadata import confirm_signal_spend_metadata, publish_signal_spend_metadata
from products.signals.backend.temporal.signal_queries import (
+ SourceSignalMatch,
+ fetch_live_source_signal_matches,
fetch_report_ids_for_scout_names,
fetch_report_ids_for_scout_prefix,
+ fetch_source_signal_matches,
retract_source_signals,
)
@@ -563,6 +571,65 @@ class TestSpendPublicationVersions(_SignalEmbeddingsTestBase):
assert confirm_signal_spend_metadata(third)
+class TestFetchLiveSourceSignalMatches(_SignalEmbeddingsTestBase):
+ def test_returns_the_latest_live_signal_for_the_exact_source(self) -> None:
+ self._emit_version(
+ document_id="older",
+ report_id="old-report",
+ source_product="github",
+ source_type="issue",
+ source_id="issue-1",
+ inserted_at=self.base,
+ )
+ self._emit_version(
+ document_id="newer",
+ report_id="new-report",
+ source_product="github",
+ source_type="issue",
+ source_id="issue-1",
+ inserted_at=self.base + timedelta(hours=1),
+ )
+ self._emit_version(
+ document_id="other-type",
+ report_id="other-report",
+ source_product="github",
+ source_type="pull_request",
+ source_id="issue-1",
+ inserted_at=self.base + timedelta(hours=2),
+ )
+
+ result = fetch_live_source_signal_matches(self.team, ["issue-1"], "github", "issue")
+
+ assert result == {
+ "issue-1": [
+ SourceSignalMatch(signal_id="newer", report_id="new-report"),
+ SourceSignalMatch(signal_id="older", report_id="old-report"),
+ ]
+ }
+
+
+class TestFetchSourceSignalMatches(APIBaseTest):
+ def test_returns_durable_association_without_retained_embedding(self) -> None:
+ report = SignalReport.objects.create(team=self.team, status=SignalReport.Status.READY)
+ SignalEmissionRecord.objects.create(
+ team=self.team,
+ source_product="github",
+ source_type="issue",
+ source_id="issue-1",
+ emitted_at=timezone.now() - timedelta(days=100),
+ signal_id="durable-signal",
+ report_id=str(report.id),
+ )
+
+ with patch(
+ "products.signals.backend.temporal.signal_queries.fetch_live_source_signal_matches"
+ ) as retained_lookup:
+ result = fetch_source_signal_matches(self.team, ["issue-1"], "github", "issue")
+
+ assert result == {"issue-1": [SourceSignalMatch(signal_id="durable-signal", report_id=str(report.id))]}
+ retained_lookup.assert_not_called()
+
+
class TestFetchSignalStatsForSourceSlice(_SignalEmbeddingsTestBase):
def test_counts_only_the_slice_and_skips_deleted_latest_versions(self) -> None:
# Guards the extra_equals pushdown and the argMax dedup: a broken JSON path would leak other